From 9ad4af1021b82cad7f8d1ac473c82d5f390d65f6 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Mon, 24 Feb 2025 19:15:59 +0300 Subject: [PATCH 01/13] activation_function --- src/.clangd | 2 ++ src/activation_function.cpp | 61 +++++++++++++++++++++++++++++++++++++ src/activation_function.h | 35 +++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 src/.clangd create mode 100644 src/activation_function.cpp create mode 100644 src/activation_function.h diff --git a/src/.clangd b/src/.clangd new file mode 100644 index 0000000..f286ea9 --- /dev/null +++ b/src/.clangd @@ -0,0 +1,2 @@ +CompileFlags: + Add: [-I/usr/include/eigen3] diff --git a/src/activation_function.cpp b/src/activation_function.cpp new file mode 100644 index 0000000..0946710 --- /dev/null +++ b/src/activation_function.cpp @@ -0,0 +1,61 @@ +#include "activation_function.h" +#include +namespace Network{ + double Sigmoid(double x) { return 1 / (1 + std::exp(-x)); } + +double SigmoidDerivative(double x) { + double sigmoid = Sigmoid(x); + return (1 - sigmoid) * sigmoid; +} + +double Tanh(double x) { return std::tanh(x); } + +double TanhDerivative(double x) { + double tanh_x = std::tanh(x); + return 1 - tanh_x * tanh_x; +} + +double ReLU(double x) { return std::max(x, 0.0); } + +double ReLUDerivative(double x) { return x > 0 ? 1 : 0; } + +ActivationFunc::ActivationFunc(NameActivationFunc name) { + switch (name) { + case NameActivationFunc::Sigmoid: + func_ = Sigmoid; + derivative_ = SigmoidDerivative; + break; + case NameActivationFunc::ReLU: + func_ = ReLU; + derivative_ = ReLUDerivative; + break; + case NameActivationFunc::Tanh: + func_ = Tanh; + derivative_ = TanhDerivative; + break; + default: + std::invalid_argument("Trouble in constructor of Activationfunc"); + } +} +Vector ActivationFunc::Activate(Vector vector){ + Vector activated_vector(vector.size()); + for(int i = 0; i +#include +#include +namespace Network { +enum class NameActivationFunc { Sigmoid, ReLU, Tanh }; +double Sigmoid(double x); +double SigmoidDerivative(double x); + +double Tanh(double x); +double TanhDerivative(double x); + +double ReLU(double x); +double ReLUDerivative(double x); +// using в своем namespace +using Vector = Eigen::VectorXd; +using Matrix = Eigen::MatrixXd; +class ActivationFunc { +public: + ActivationFunc() = default; + + // Я вспомнил пример про размеры картинок и не захотелась передавать 2 + // std::function. С другой стороны тут функции активации, котороые мы можем передовать строго фиксированы и пользователь не сможет передать свою функцию. + + ActivationFunc(NameActivationFunc name); + Vector Activate(Vector vector); + Matrix GetDifferential(Vector vector); + +private: + // или лучше функтор/стирающиеся типы? + std::function func_; + std::function derivative_; +}; + +} From a76e442269a4d9d488a83f6bb771508944867dc4 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Fri, 14 Mar 2025 00:08:09 +0300 Subject: [PATCH 02/13] Sorry, I did not fix python-style in ActivationFunc and LossFunc. I will do it soonls --- .clang-format | 10 +++ .clang-tidy | 41 +++++++++ .gitignore | 13 ++- .gitmodules | 7 ++ CMakeLists.txt | 50 +++++++++++ libs/EigenRand | 1 + libs/googletest | 1 + src/.clangd | 2 - src/activation_function.cpp | 128 ++++++++++++++++------------- src/activation_function.h | 60 ++++++++------ src/layer.cpp | 4 + src/layer.h | 27 ++++++ src/linalg.h | 9 ++ src/loss_function.cpp | 57 +++++++++++++ src/loss_function.h | 30 +++++++ src/main.cpp | 17 ++++ src/net.cpp | 0 src/net.h | 0 tests/test_activation_function.cpp | 23 ++++++ tests/test_layer.cpp | 0 tests/test_loss_function.cpp | 0 21 files changed, 391 insertions(+), 89 deletions(-) create mode 100644 .clang-format create mode 100644 .clang-tidy create mode 100644 .gitmodules create mode 100644 CMakeLists.txt create mode 160000 libs/EigenRand create mode 160000 libs/googletest delete mode 100644 src/.clangd create mode 100644 src/layer.cpp create mode 100644 src/layer.h create mode 100644 src/linalg.h create mode 100644 src/loss_function.cpp create mode 100644 src/loss_function.h create mode 100644 src/main.cpp create mode 100644 src/net.cpp create mode 100644 src/net.h create mode 100644 tests/test_activation_function.cpp create mode 100644 tests/test_layer.cpp create mode 100644 tests/test_loss_function.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..fe699e0 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: Google +IndentWidth: 4 +AccessModifierOffset: -4 +ColumnLimit: 100 +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +DerivePointerAlignment: true +KeepEmptyLinesAtTheStartOfBlocks: true +SortIncludes: false diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..3d705b4 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,41 @@ +--- +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +HeaderFilterRegex: '\.h$' +WarningsAsErrors: '*' +CheckOptions: + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.TypedefCase + value: CamelCase + - key: readability-identifier-naming.TypedefIgnoredRegexp + value: (allocator_type|size_type|key_type|value_type|mapped_type|difference_type|pointer|const_pointer|reference|const_reference|iterator_category|const_iterator|iterator|reverse_iterator|const_reverse_iterator|key_compare) + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.PrivateMemberSuffix + value: '_' + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: CamelCase + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantPrefix + value: k + - key: readability-identifier-naming.GlobalConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: google-runtime-int.TypeSuffix + value: _t \ No newline at end of file diff --git a/.gitignore b/.gitignore index 259148f..b7d0d98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,31 @@ -# Prerequisites *.d -# Compiled Object files *.slo *.lo *.o *.obj -# Precompiled Headers *.gch *.pch -# Compiled Dynamic libraries *.so *.dylib *.dll -# Fortran module files *.mod *.smod -# Compiled Static libraries *.lai *.la *.a *.lib -# Executables *.exe *.out *.app + +compile_commands.json +.cache* +.vscode* +build* +libs/eigen \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3ec7fd0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,7 @@ + +[submodule "libs/EigenRand"] + path = libs/EigenRand + url = https://github.com/bab2min/EigenRand +[submodule "libs/googletest"] + path = libs/googletest + url = https://github.com/google/googletest.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ef08bab --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.10) +project(NeuralNetwork LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + + +include_directories( + ${CMAKE_SOURCE_DIR}/libs/eigen + ${CMAKE_SOURCE_DIR}/libs/EigenRand + ${CMAKE_SOURCE_DIR}/libs/googletest/googletest/include + ${CMAKE_SOURCE_DIR}/src +) + + +add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) +add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) +add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) + +add_executable(test_activation_function + tests/test_activation_function.cpp + src/activation_function.cpp # Добавляем исходный файл с реализацией +) +target_link_libraries(test_activation_function PRIVATE gtest gtest_main) + + +add_executable(test_loss_function + tests/test_loss_function.cpp + src/loss_function.cpp +) +target_link_libraries(test_loss_function PRIVATE gtest gtest_main) + +add_executable(test_layer + tests/test_layer.cpp + src/layer.cpp) +target_link_libraries(test_layer PRIVATE gtest gtest_main) + +add_executable(main src/main.cpp) + + + +enable_testing() +add_test(NAME test_activation_function COMMAND test_activation_function) +add_test(NAME test_loss_function COMMAND test_loss_function) +add_test(NAME test_layer COMMAND test_layer) + + + diff --git a/libs/EigenRand b/libs/EigenRand new file mode 160000 index 0000000..f3190cd --- /dev/null +++ b/libs/EigenRand @@ -0,0 +1 @@ +Subproject commit f3190cd7f85c5878c949625cb2811262ff050d66 diff --git a/libs/googletest b/libs/googletest new file mode 160000 index 0000000..4902ea2 --- /dev/null +++ b/libs/googletest @@ -0,0 +1 @@ +Subproject commit 4902ea2d7c6faed89b6facee00baa34bb108fc0d diff --git a/src/.clangd b/src/.clangd deleted file mode 100644 index f286ea9..0000000 --- a/src/.clangd +++ /dev/null @@ -1,2 +0,0 @@ -CompileFlags: - Add: [-I/usr/include/eigen3] diff --git a/src/activation_function.cpp b/src/activation_function.cpp index 0946710..bcf715d 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -1,61 +1,79 @@ + #include "activation_function.h" -#include -namespace Network{ - double Sigmoid(double x) { return 1 / (1 + std::exp(-x)); } - -double SigmoidDerivative(double x) { - double sigmoid = Sigmoid(x); - return (1 - sigmoid) * sigmoid; -} - -double Tanh(double x) { return std::tanh(x); } - -double TanhDerivative(double x) { - double tanh_x = std::tanh(x); - return 1 - tanh_x * tanh_x; -} - -double ReLU(double x) { return std::max(x, 0.0); } - -double ReLUDerivative(double x) { return x > 0 ? 1 : 0; } - -ActivationFunc::ActivationFunc(NameActivationFunc name) { - switch (name) { - case NameActivationFunc::Sigmoid: - func_ = Sigmoid; - derivative_ = SigmoidDerivative; - break; - case NameActivationFunc::ReLU: - func_ = ReLU; - derivative_ = ReLUDerivative; - break; - case NameActivationFunc::Tanh: - func_ = Tanh; - derivative_ = TanhDerivative; - break; - default: - std::invalid_argument("Trouble in constructor of Activationfunc"); - } -} -Vector ActivationFunc::Activate(Vector vector){ - Vector activated_vector(vector.size()); - for(int i = 0; i +namespace network { +namespace details_activation_functions { +Vector Sigmoid::Activate(const Vector &vector) { + return vector.unaryExpr([](double x) { return 1.0 / (1.0 + std::exp(-x)); }); +} +Matrix Sigmoid::GetDifferential(const Vector &vector) { + Vector activated_vector = Activate(vector); + Vector diff = activated_vector.unaryExpr([](double x) { return (1.0 - x) * x; }); + return diff.asDiagonal(); +} + +Vector Tanh::Activate(const Vector &vector) { + return vector.unaryExpr([](double x) { return std::tanh(x); }); +} + +Matrix Tanh::GetDifferential(const Vector &vector) { + Vector activated_vector = Activate(vector); + Vector diff = activated_vector.unaryExpr([](double x) { return 1.0 - x * x; }); + return diff.asDiagonal(); +} +Vector ReLU::Activate(const Vector &vector) { + return vector.unaryExpr([](double x) { return std::max(0.0, x); }); } +Matrix ReLU::GetDifferential(const Vector &vector) { + Vector diff = vector.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); + return diff.asDiagonal(); } +Vector Softmax::Activate(const Vector &vector) { + Vector tmp = vector.unaryExpr([](double x) { return std::exp(x); }); + Vector activated_vector = tmp / tmp.sum(); + return activated_vector; +} +Matrix Softmax::GetDifferential(const Vector &vector) { + Vector tmp = Activate(vector); + Matrix jacobian(tmp.size(), tmp.size()); + for (int i = 0; i < tmp.size(); ++i) { + for (int j = 0; j < tmp.size(); ++j) { + jacobian(i, j) = tmp(i) * ((i == j) ? (1.0 - tmp(j)) : -tmp(j)); + } + } + return jacobian; +} + +} // namespace details_activation_functions + +ActivationFunc::ActivationFunc(NamesActivationFunc name) { + switch (name) { + case NamesActivationFunc::Sigmoid: + apply_ = details_activation_functions::Sigmoid::Activate; + differential_ = details_activation_functions::Sigmoid::GetDifferential; + break; + case NamesActivationFunc::ReLU: + apply_ = details_activation_functions::ReLU::Activate; + differential_ = details_activation_functions::ReLU::GetDifferential; + break; + case NamesActivationFunc::Tanh: + apply_ = details_activation_functions::Tanh::Activate; + differential_ = details_activation_functions::Tanh::GetDifferential; + break; + case NamesActivationFunc::Softmax: + apply_ = details_activation_functions::Softmax::Activate; + differential_ = details_activation_functions::Softmax::GetDifferential; + default: + assert("invalid arguments in ActivationFunc constructor"); + } +} +Vector ActivationFunc::Activate(const Vector &vector) { + return apply_(vector); +} +Matrix ActivationFunc::GetDifferential(const Vector &vector) { + return differential_(vector); +} +} // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h index 1069e6b..d49750a 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -1,35 +1,45 @@ #pragma once -#include +#include "linalg.h" #include #include -namespace Network { -enum class NameActivationFunc { Sigmoid, ReLU, Tanh }; -double Sigmoid(double x); -double SigmoidDerivative(double x); - -double Tanh(double x); -double TanhDerivative(double x); - -double ReLU(double x); -double ReLUDerivative(double x); -// using в своем namespace -using Vector = Eigen::VectorXd; -using Matrix = Eigen::MatrixXd; +namespace network { +enum class NamesActivationFunc { Sigmoid, ReLU, Tanh, Softmax }; + +namespace details_activation_functions { +struct Sigmoid { + static Vector Activate(const Vector& vector); + static Matrix GetDifferential(const Vector& vector); +}; + +struct Tanh { + static Vector Activate(const Vector& vector); + static Matrix GetDifferential(const Vector& vector); +}; + +struct ReLU { + static Vector Activate(const Vector& vector); + static Matrix GetDifferential(const Vector& vector); +}; +struct Softmax { + static Vector Activate(const Vector& vector); + static Matrix GetDifferential(const Vector& vector); +}; +} // namespace details_activation_functions + class ActivationFunc { -public: - ActivationFunc() = default; + using Function = std::function; + using Differential = std::function; + //ActivationFunc(Function&& apply, Differential&& differential); - // Я вспомнил пример про размеры картинок и не захотелась передавать 2 - // std::function. С другой стороны тут функции активации, котороые мы можем передовать строго фиксированы и пользователь не сможет передать свою функцию. - ActivationFunc(NameActivationFunc name); - Vector Activate(Vector vector); - Matrix GetDifferential(Vector vector); +public: + explicit ActivationFunc(NamesActivationFunc name); + Vector Activate(const Vector& vector); + Matrix GetDifferential(const Vector& vector); private: - // или лучше функтор/стирающиеся типы? - std::function func_; - std::function derivative_; + Function apply_; + Differential differential_; }; -} +} // namespace network diff --git a/src/layer.cpp b/src/layer.cpp new file mode 100644 index 0000000..36205a8 --- /dev/null +++ b/src/layer.cpp @@ -0,0 +1,4 @@ +#include "layer.h" +namespace network{ + +} \ No newline at end of file diff --git a/src/layer.h b/src/layer.h new file mode 100644 index 0000000..d8b68b1 --- /dev/null +++ b/src/layer.h @@ -0,0 +1,27 @@ +#include "activation_function.h" + +#include "linalg.h" + +namespace network{ + + class Layer{ + public: + + void Linear(int input_size, int output_size); + void SetActivationFuntion(NamesActivationFunc name); + Vector Forward(const Vector& input); + Matrix Forward(const Matrix& input); + Vector Backward(const Vector& gradient); + Matrix Backward(const Matrix& gradient); + + + + private: + ActivationFunc func_; + Matrix weigts_; + Vector bias_; + bool is_set_func_; + void SetRandomWeigts(); + }; + +} \ No newline at end of file diff --git a/src/linalg.h b/src/linalg.h new file mode 100644 index 0000000..9980684 --- /dev/null +++ b/src/linalg.h @@ -0,0 +1,9 @@ +#include +#include + +namespace network { +using Matrix = Eigen::MatrixXd; +using Vector = Eigen::VectorXd; +using VectorT = Eigen::RowVectorXd; + +} // namespace network \ No newline at end of file diff --git a/src/loss_function.cpp b/src/loss_function.cpp new file mode 100644 index 0000000..b3494fa --- /dev/null +++ b/src/loss_function.cpp @@ -0,0 +1,57 @@ +#include "loss_function.h" + +namespace network { + +double Mse::GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).array().square().mean(); +} +Vector Mse::GetGradient(const Vector &y_out, const Vector &y_expected) { + + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return (2.0 / y_out.size()) * (y_out - y_expected); +} +double CrossEntropy::GetValue(const Vector &y_out, const Vector &y_expected) { + + assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); + const double eps = 1e-12; + return -(y_expected.array() * (y_out.array() + eps).log()).sum(); +} +Vector CrossEntropy::GetGradient(const Vector &y_out, + const Vector &y_expected) { + + assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); + const double eps = 1e-12; + return -(y_expected.array() / (y_out.array() + eps)); +} +LossFunc::LossFunc(NamesLossFunctions name) { + switch (name) { + case NamesLossFunctions::Mse: + loss_func_ = Mse::GetValue; + get_grad_ = Mse::GetGradient; + case network::NamesLossFunctions::CrossEntorpy: + loss_func_ = CrossEntropy::GetValue; + get_grad_ = CrossEntropy::GetGradient; + default: + assert("Problen in LossFunc constructor"); + } +} +double LossFunc::Dist(const Vector &y_out, const Vector &y_expected) { + return loss_func_(y_out, y_expected); +} +VectorT LossFunc::GetGradient(const Vector &y_out, const Vector &y_expected) { + return get_grad_(y_out, y_expected).transpose(); +} +Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) { + assert(y_out.cols() == y_expected.cols() && + y_out.rows() == y_expected.rows() && + "Matrices have different sizes. GetGradient."); + Matrix res(y_out.rows(), y_out.cols()); + for (int i = 0; i < y_out.cols(); ++i) { + VectorT grad_of_one_vector = + res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); + } + return res; +} + +} // namespace Network diff --git a/src/loss_function.h b/src/loss_function.h new file mode 100644 index 0000000..04a9b6a --- /dev/null +++ b/src/loss_function.h @@ -0,0 +1,30 @@ +#pragma once +#include "linalg.h" +#include +namespace network { + + +enum class NamesLossFunctions { Mse, CrossEntorpy }; + +struct Mse { + static double GetValue(const Vector &y_out, const Vector &y_expected); + static Vector GetGradient(const Vector &y_out, const Vector &y_expected); +}; +struct CrossEntropy { + static double GetValue(const Vector &y_out, const Vector &y_expected); + static Vector GetGradient(const Vector &y_out, const Vector &y_expected); +}; + +class LossFunc { + public: + LossFunc(NamesLossFunctions name); + double Dist(const Vector &y_out, const Vector &y_expected); + VectorT GetGradient(const Vector &y_out, const Vector &y_expected); + Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); + + private: + std::function loss_func_; + std::function get_grad_; + }; + +} // namespace Network \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..c6299ac --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,17 @@ +#include +#include +#include +#include +using RandGen = Eigen::Rand::Vmt19937_64; + +RandGen& GetRng() { + static RandGen rng = 1; + return rng; +} +int main() { + int rows = 2; + int cols = 2; + Eigen::MatrixXd result = Eigen::Rand::normal(rows, cols, GetRng()); + std::cout << result << std::endl; + return 0; +} \ No newline at end of file diff --git a/src/net.cpp b/src/net.cpp new file mode 100644 index 0000000..e69de29 diff --git a/src/net.h b/src/net.h new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_activation_function.cpp b/tests/test_activation_function.cpp new file mode 100644 index 0000000..c53dfc3 --- /dev/null +++ b/tests/test_activation_function.cpp @@ -0,0 +1,23 @@ +#include +#include "activation_function.h" + + + + + + +TEST(ActivationfuncTest, Relu){ + network::ActivationFunc func(network::NamesActivationFunc::ReLU); + Eigen::VectorXd vector(2); + vector[1] = 1; + vector[0] = 1; + Eigen::MatrixXd g = func.Activate(vector); + EXPECT_EQ(vector, g); + +} + + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tests/test_layer.cpp b/tests/test_layer.cpp new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_loss_function.cpp b/tests/test_loss_function.cpp new file mode 100644 index 0000000..e69de29 From faedbe700e227153ba46fbee1a6b3cb9727d5e41 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Fri, 14 Mar 2025 00:11:10 +0300 Subject: [PATCH 03/13] Sorry, I did not fix python-style in ActivationFunc and LossFunc. I will do it soon --- src/activation_function.cpp | 1 - src/activation_function.h | 17 +++++----- src/layer.cpp | 4 +-- src/layer.h | 37 ++++++++++---------- src/loss_function.cpp | 67 ++++++++++++++++++------------------- src/loss_function.h | 21 ++++++------ 6 files changed, 68 insertions(+), 79 deletions(-) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index bcf715d..8e7597b 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -48,7 +48,6 @@ Matrix Softmax::GetDifferential(const Vector &vector) { } // namespace details_activation_functions - ActivationFunc::ActivationFunc(NamesActivationFunc name) { switch (name) { case NamesActivationFunc::Sigmoid: diff --git a/src/activation_function.h b/src/activation_function.h index d49750a..b923616 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -27,19 +27,18 @@ struct Softmax { } // namespace details_activation_functions class ActivationFunc { - using Function = std::function; - using Differential = std::function; - //ActivationFunc(Function&& apply, Differential&& differential); - + using Function = std::function; + using Differential = std::function; + // ActivationFunc(Function&& apply, Differential&& differential); public: - explicit ActivationFunc(NamesActivationFunc name); - Vector Activate(const Vector& vector); - Matrix GetDifferential(const Vector& vector); + explicit ActivationFunc(NamesActivationFunc name); + Vector Activate(const Vector& vector); + Matrix GetDifferential(const Vector& vector); private: - Function apply_; - Differential differential_; + Function apply_; + Differential differential_; }; } // namespace network diff --git a/src/layer.cpp b/src/layer.cpp index 36205a8..28ce4ec 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,4 +1,2 @@ #include "layer.h" -namespace network{ - -} \ No newline at end of file +namespace network {} \ No newline at end of file diff --git a/src/layer.h b/src/layer.h index d8b68b1..a204d65 100644 --- a/src/layer.h +++ b/src/layer.h @@ -2,26 +2,23 @@ #include "linalg.h" -namespace network{ +namespace network { - class Layer{ - public: - - void Linear(int input_size, int output_size); - void SetActivationFuntion(NamesActivationFunc name); - Vector Forward(const Vector& input); - Matrix Forward(const Matrix& input); - Vector Backward(const Vector& gradient); - Matrix Backward(const Matrix& gradient); +class Layer { +public: + void Linear(int input_size, int output_size); + void SetActivationFuntion(NamesActivationFunc name); + Vector Forward(const Vector& input); + Matrix Forward(const Matrix& input); + Vector Backward(const Vector& gradient); + Matrix Backward(const Matrix& gradient); - +private: + ActivationFunc func_; + Matrix weigts_; + Vector bias_; + bool is_set_func_; + void SetRandomWeigts(); +}; - private: - ActivationFunc func_; - Matrix weigts_; - Vector bias_; - bool is_set_func_; - void SetRandomWeigts(); - }; - -} \ No newline at end of file +} // namespace network \ No newline at end of file diff --git a/src/loss_function.cpp b/src/loss_function.cpp index b3494fa..a862340 100644 --- a/src/loss_function.cpp +++ b/src/loss_function.cpp @@ -1,57 +1,54 @@ #include "loss_function.h" namespace network { - + double Mse::GetValue(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "Mse GetValue"); - return (y_out - y_expected).array().square().mean(); + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).array().square().mean(); } Vector Mse::GetGradient(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "Mse GetGradient"); - return (2.0 / y_out.size()) * (y_out - y_expected); + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return (2.0 / y_out.size()) * (y_out - y_expected); } double CrossEntropy::GetValue(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); - const double eps = 1e-12; - return -(y_expected.array() * (y_out.array() + eps).log()).sum(); + assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); + const double eps = 1e-12; + return -(y_expected.array() * (y_out.array() + eps).log()).sum(); } -Vector CrossEntropy::GetGradient(const Vector &y_out, - const Vector &y_expected) { +Vector CrossEntropy::GetGradient(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); - const double eps = 1e-12; - return -(y_expected.array() / (y_out.array() + eps)); + assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); + const double eps = 1e-12; + return -(y_expected.array() / (y_out.array() + eps)); } LossFunc::LossFunc(NamesLossFunctions name) { - switch (name) { - case NamesLossFunctions::Mse: - loss_func_ = Mse::GetValue; - get_grad_ = Mse::GetGradient; - case network::NamesLossFunctions::CrossEntorpy: - loss_func_ = CrossEntropy::GetValue; - get_grad_ = CrossEntropy::GetGradient; - default: - assert("Problen in LossFunc constructor"); - } + switch (name) { + case NamesLossFunctions::Mse: + loss_func_ = Mse::GetValue; + get_grad_ = Mse::GetGradient; + case network::NamesLossFunctions::CrossEntorpy: + loss_func_ = CrossEntropy::GetValue; + get_grad_ = CrossEntropy::GetGradient; + default: + assert("Problen in LossFunc constructor"); + } } double LossFunc::Dist(const Vector &y_out, const Vector &y_expected) { - return loss_func_(y_out, y_expected); + return loss_func_(y_out, y_expected); } VectorT LossFunc::GetGradient(const Vector &y_out, const Vector &y_expected) { - return get_grad_(y_out, y_expected).transpose(); + return get_grad_(y_out, y_expected).transpose(); } Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) { - assert(y_out.cols() == y_expected.cols() && - y_out.rows() == y_expected.rows() && - "Matrices have different sizes. GetGradient."); - Matrix res(y_out.rows(), y_out.cols()); - for (int i = 0; i < y_out.cols(); ++i) { - VectorT grad_of_one_vector = - res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); - } - return res; + assert(y_out.cols() == y_expected.cols() && y_out.rows() == y_expected.rows() && + "Matrices have different sizes. GetGradient."); + Matrix res(y_out.rows(), y_out.cols()); + for (int i = 0; i < y_out.cols(); ++i) { + VectorT grad_of_one_vector = res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); + } + return res; } -} // namespace Network +} // namespace network diff --git a/src/loss_function.h b/src/loss_function.h index 04a9b6a..6247c97 100644 --- a/src/loss_function.h +++ b/src/loss_function.h @@ -3,28 +3,27 @@ #include namespace network { - enum class NamesLossFunctions { Mse, CrossEntorpy }; struct Mse { - static double GetValue(const Vector &y_out, const Vector &y_expected); - static Vector GetGradient(const Vector &y_out, const Vector &y_expected); + static double GetValue(const Vector &y_out, const Vector &y_expected); + static Vector GetGradient(const Vector &y_out, const Vector &y_expected); }; struct CrossEntropy { - static double GetValue(const Vector &y_out, const Vector &y_expected); - static Vector GetGradient(const Vector &y_out, const Vector &y_expected); + static double GetValue(const Vector &y_out, const Vector &y_expected); + static Vector GetGradient(const Vector &y_out, const Vector &y_expected); }; class LossFunc { - public: +public: LossFunc(NamesLossFunctions name); double Dist(const Vector &y_out, const Vector &y_expected); VectorT GetGradient(const Vector &y_out, const Vector &y_expected); Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); - - private: + +private: std::function loss_func_; std::function get_grad_; - }; - -} // namespace Network \ No newline at end of file +}; + +} // namespace network \ No newline at end of file From b84dae7eb91d37e6e63e187d3661e7e39d2c5c28 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Fri, 21 Mar 2025 19:33:01 +0300 Subject: [PATCH 04/13] added adam_optimizer and layer --- .gitignore | 2 +- CMakeLists.txt | 9 +- src/activation_function.cpp | 133 ++++++++++++++++------------- src/activation_function.h | 37 ++------ src/adam_optimizer.cpp | 29 +++++++ src/adam_optimizer.h | 23 +++++ src/dataloader.cpp | 78 +++++++++++++++++ src/dataloader.h | 40 +++++++++ src/layer.cpp | 98 ++++++++++++++++++++- src/layer.h | 56 ++++++++++-- src/linalg.h | 2 + src/loss_function.cpp | 60 +++++++------ src/loss_function.h | 18 +--- src/main.cpp | 17 +--- src/net.cpp | 23 +++++ src/net.h | 24 ++++++ tests/test_activation_function.cpp | 33 ++++--- tests/test_dataloader.cpp | 18 ++++ tests/test_layer.cpp | 57 +++++++++++++ tests/test_loss_function.cpp | 2 + 20 files changed, 587 insertions(+), 172 deletions(-) create mode 100644 src/adam_optimizer.cpp create mode 100644 src/adam_optimizer.h create mode 100644 src/dataloader.cpp create mode 100644 src/dataloader.h create mode 100644 tests/test_dataloader.cpp diff --git a/.gitignore b/.gitignore index b7d0d98..483404a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,4 @@ compile_commands.json .cache* .vscode* build* -libs/eigen \ No newline at end of file +libs/eigen diff --git a/CMakeLists.txt b/CMakeLists.txt index ef08bab..b15d913 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ include_directories( add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) -add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) -add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) +# add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) +# add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) add_executable(test_activation_function tests/test_activation_function.cpp @@ -34,10 +34,11 @@ target_link_libraries(test_loss_function PRIVATE gtest gtest_main) add_executable(test_layer tests/test_layer.cpp - src/layer.cpp) + src/layer.cpp + src/activation_function.cpp) target_link_libraries(test_layer PRIVATE gtest gtest_main) -add_executable(main src/main.cpp) +add_executable(main src/main.cpp src/adam_optimizer.cpp) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index 8e7597b..5233528 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -2,77 +2,96 @@ #include "activation_function.h" #include namespace network { -namespace details_activation_functions { -Vector Sigmoid::Activate(const Vector &vector) { - return vector.unaryExpr([](double x) { return 1.0 / (1.0 + std::exp(-x)); }); -} - -Matrix Sigmoid::GetDifferential(const Vector &vector) { - Vector activated_vector = Activate(vector); - Vector diff = activated_vector.unaryExpr([](double x) { return (1.0 - x) * x; }); - return diff.asDiagonal(); -} - -Vector Tanh::Activate(const Vector &vector) { - return vector.unaryExpr([](double x) { return std::tanh(x); }); -} +namespace details { +struct Sigmoid { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](double x) { return 1.0 / (1.0 + std::exp(-x)); }); + } + static Matrix GetDifferential(const Vector& x) { + Vector activated_vector = Apply(x); + Vector diff = activated_vector.unaryExpr([](double x) { return (1.0 - x) * x; }); + return diff.asDiagonal(); + } +}; -Matrix Tanh::GetDifferential(const Vector &vector) { - Vector activated_vector = Activate(vector); - Vector diff = activated_vector.unaryExpr([](double x) { return 1.0 - x * x; }); - return diff.asDiagonal(); -} -Vector ReLU::Activate(const Vector &vector) { - return vector.unaryExpr([](double x) { return std::max(0.0, x); }); -} +struct Tanh { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](double x) { return std::tanh(x); }); + } + static Matrix GetDifferential(const Vector& x) { + Vector activated_vector = Apply(x); + Vector diff = activated_vector.unaryExpr([](double x) { return 1.0 - x * x; }); + return diff.asDiagonal(); + } +}; -Matrix ReLU::GetDifferential(const Vector &vector) { - Vector diff = vector.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); - return diff.asDiagonal(); -} -Vector Softmax::Activate(const Vector &vector) { - Vector tmp = vector.unaryExpr([](double x) { return std::exp(x); }); - Vector activated_vector = tmp / tmp.sum(); - return activated_vector; -} -Matrix Softmax::GetDifferential(const Vector &vector) { - Vector tmp = Activate(vector); - Matrix jacobian(tmp.size(), tmp.size()); - for (int i = 0; i < tmp.size(); ++i) { - for (int j = 0; j < tmp.size(); ++j) { - jacobian(i, j) = tmp(i) * ((i == j) ? (1.0 - tmp(j)) : -tmp(j)); +struct ReLU { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](double x) { return std::max(0.0, x); }); + } + static Matrix GetDifferential(const Vector& x) { + Vector diff = x.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); + return diff.asDiagonal(); + } +}; +struct Softmax { + static Vector Apply(const Vector& x) { + Vector tmp = x.unaryExpr([](double x) { return std::exp(x); }); + Vector activated_vector = tmp / tmp.sum(); + return activated_vector; + } + static Matrix GetDifferential(const Vector& x) { + Vector tmp = Apply(x); + Matrix jacobian(tmp.size(), tmp.size()); + for (int i = 0; i < tmp.size(); ++i) { + for (int j = 0; j < tmp.size(); ++j) { + jacobian(i, j) = tmp(i) * ((i == j) ? (1.0 - tmp(j)) : -tmp(j)); + } } + return jacobian; } - return jacobian; -} +}; -} // namespace details_activation_functions +} // namespace details -ActivationFunc::ActivationFunc(NamesActivationFunc name) { +void ActivationFunc::SetFunction(ActivationFunc::Name name) { switch (name) { - case NamesActivationFunc::Sigmoid: - apply_ = details_activation_functions::Sigmoid::Activate; - differential_ = details_activation_functions::Sigmoid::GetDifferential; + case ActivationFunc::Name::Sigmoid: + apply_ = details::Sigmoid::Apply; + differential_ = details::Sigmoid::GetDifferential; break; - case NamesActivationFunc::ReLU: - apply_ = details_activation_functions::ReLU::Activate; - differential_ = details_activation_functions::ReLU::GetDifferential; + case ActivationFunc::Name::ReLU: + apply_ = details::ReLU::Apply; + differential_ = details::ReLU::GetDifferential; break; - case NamesActivationFunc::Tanh: - apply_ = details_activation_functions::Tanh::Activate; - differential_ = details_activation_functions::Tanh::GetDifferential; + case ActivationFunc::Name::Tanh: + apply_ = details::Tanh::Apply; + differential_ = details::Tanh::GetDifferential; break; - case NamesActivationFunc::Softmax: - apply_ = details_activation_functions::Softmax::Activate; - differential_ = details_activation_functions::Softmax::GetDifferential; + case ActivationFunc::Name::Softmax: + apply_ = details::Softmax::Apply; + differential_ = details::Softmax::GetDifferential; default: - assert("invalid arguments in ActivationFunc constructor"); + assert(false && "invalid arguments in ActivationFunc constructor"); } } -Vector ActivationFunc::Activate(const Vector &vector) { + +ActivationFunc::ActivationFunc(ActivationFunc::Name name) { + SetFunction(name); +} +Vector ActivationFunc::Apply(const Vector& vector) const { return apply_(vector); } -Matrix ActivationFunc::GetDifferential(const Vector &vector) { - return differential_(vector); +Matrix ActivationFunc::Apply(const Matrix& x) const { + Matrix res(x.rows(), x.cols()); + + for (int i = 0; i < x.cols(); ++i) { + res.col(i) = apply_(x.col(i)); + } + + return res; +} +Matrix ActivationFunc::GetDifferential(const Vector& x) const { + return differential_(x); } } // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h index b923616..0dea46a 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -3,38 +3,17 @@ #include #include namespace network { -enum class NamesActivationFunc { Sigmoid, ReLU, Tanh, Softmax }; - -namespace details_activation_functions { -struct Sigmoid { - static Vector Activate(const Vector& vector); - static Matrix GetDifferential(const Vector& vector); -}; - -struct Tanh { - static Vector Activate(const Vector& vector); - static Matrix GetDifferential(const Vector& vector); -}; - -struct ReLU { - static Vector Activate(const Vector& vector); - static Matrix GetDifferential(const Vector& vector); -}; -struct Softmax { - static Vector Activate(const Vector& vector); - static Matrix GetDifferential(const Vector& vector); -}; -} // namespace details_activation_functions - class ActivationFunc { - using Function = std::function; - using Differential = std::function; - // ActivationFunc(Function&& apply, Differential&& differential); + using Function = std::function; + using Differential = std::function; public: - explicit ActivationFunc(NamesActivationFunc name); - Vector Activate(const Vector& vector); - Matrix GetDifferential(const Vector& vector); + enum class Name { Sigmoid, ReLU, Tanh, Softmax }; + explicit ActivationFunc(Name name); + void SetFunction(Name name); + Vector Apply(const Vector& x) const; + Matrix Apply(const Matrix& x) const; + Matrix GetDifferential(const Vector& x) const; private: Function apply_; diff --git a/src/adam_optimizer.cpp b/src/adam_optimizer.cpp new file mode 100644 index 0000000..1bde409 --- /dev/null +++ b/src/adam_optimizer.cpp @@ -0,0 +1,29 @@ +#include "adam_optimizer.h" +#include + +namespace network { + +AdamOptimizer::AdamOptimizer(const Matrix& parametr, double start_learning_rate, double beta1, + double beta2) + : t_(0), + learning_rate_(start_learning_rate), + beta1_(beta1), + beta2_(beta2), + m_t_(Matrix::Zero(parametr.rows(), parametr.cols())), + v_t_(Matrix::Zero(parametr.rows(), parametr.cols())) { +} + +Matrix AdamOptimizer::ComputeUpdate(const Matrix& gradient) { + assert(gradient.rows() == m_t_.rows() && gradient.cols() && m_t_.cols() && "invalid gradient"); + t_ += 1; + m_t_ = beta1_ * m_t_ + (1 - beta1_) * gradient; + Matrix squared_gradient = gradient.cwiseProduct(gradient); + v_t_ = beta2_ * v_t_ + (1 - beta2_) * squared_gradient; + double m_bias_correction = 1.0 - std::pow(beta1_, t_); + double v_bias_correction = 1.0 - std::pow(beta2_, t_); + Matrix m_tmp = m_t_ / m_bias_correction; + Matrix v_tmp = v_t_ / v_bias_correction; + return learning_rate_ * m_tmp.array() / (v_tmp.array().sqrt() + kEps); +} + +} // namespace network diff --git a/src/adam_optimizer.h b/src/adam_optimizer.h new file mode 100644 index 0000000..b9efd00 --- /dev/null +++ b/src/adam_optimizer.h @@ -0,0 +1,23 @@ +#pragma once +#include "linalg.h" + +namespace network { +class AdamOptimizer { +public: + AdamOptimizer(const Matrix& parametr, double start_learning_rate = kDefaultStartLearningRate, + double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); + Matrix ComputeUpdate(const Matrix& gradient); + +private: + static constexpr double kDefaultStartLearningRate = 3e-4; + static constexpr double kDefaultBeta1 = 0.9; + static constexpr double kDefaultBeta2 = 0.99; + static constexpr double kEps = 1e-8; + int t_; + double learning_rate_; + double beta1_; + double beta2_; + Matrix m_t_; + Matrix v_t_; +}; +} // namespace network \ No newline at end of file diff --git a/src/dataloader.cpp b/src/dataloader.cpp new file mode 100644 index 0000000..67f131d --- /dev/null +++ b/src/dataloader.cpp @@ -0,0 +1,78 @@ +#include "dataloader.h" +#include +#include +#include +#include + +namespace network { +namespace details { +Shuffle::Shuffle() : gen_(kDefaultSeed) { +} +Shuffle::Shuffle(int seed) : gen_(kDefaultSeed) { +} +void Shuffle::ShuffleData(Index begin, Index end, Data& data) { + assert(begin >= 0 && "Negative start index"); + assert(end > begin && "Invalid range"); + assert(end <= data.input.cols() && "Range exceeds matrix columns"); + assert(data.input.cols() == data.output.cols() && "Matrix size mismatch"); + + for (Index i = end - 1; i > begin; --i) { + std::uniform_int_distribution uni(begin, i); + Index j = uni(gen_); + data.input.col(i).swap(data.input.col(j)); + data.output.col(i).swap(data.output.col(j)); + } +} + +} // namespace details + +DataLoader::DataLoader(Data&& data) { + assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); + data_ = std::move(data); + ShuffleData(); +} + +DataLoader::DataLoader(const Data& data) { + assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); + data_ = data; + ShuffleData(); +} +int DataLoader::Size() const { + return data_.input.cols(); +} + +std::vector DataLoader::Batches(int batch_size) const { + assert(batch_size > 0 && "Batch size must be positive"); + assert(batch_size <= Size() && "Batch size exceeds training data size"); + std::vector batches; + int data_size = Size(); + for (int i = 0; i < data_size; i += batch_size) { + int cur_batch_size = std::min(batch_size, data_size - i); + batches.push_back(GetBatch(i, cur_batch_size)); + } + return batches; +} + +// namespace + +void DataLoader::ShuffleData(Shuffle& rnd) { + rnd.ShuffleData(0, Size(), data_); +} +DataLoader::Shuffle& DataLoader::GlobalShuffle() { + static Shuffle rnd; + return rnd; +} +Data DataLoader::GetData() const { + return data_; +} + +Data DataLoader::GetBatch(Index begin, int size) const { + assert(begin >= 0 && size > 0 && "Invalid batch parameters"); + assert(begin + size <= Size() && "Batch exceeds training data"); + Data batch; + batch.input = data_.input.middleCols(begin, size); + batch.output = data_.output.middleCols(begin, size); + return batch; +} + +} // namespace network \ No newline at end of file diff --git a/src/dataloader.h b/src/dataloader.h new file mode 100644 index 0000000..e60ef53 --- /dev/null +++ b/src/dataloader.h @@ -0,0 +1,40 @@ +#pragma once +#include "linalg.h" +namespace network { + +struct Data { + Matrix input; + Matrix output; +}; +namespace details { + +class Shuffle { + +public: + Shuffle(); + Shuffle(int seed); + void ShuffleData(Index begin, Index end, Data& data); + +private: + static constexpr int kDefaultSeed = 42; + std::mt19937 gen_; +}; +} // namespace details +class DataLoader { + using Shuffle = details::Shuffle; + +public: + DataLoader(const Data& data); + DataLoader(Data&& data); + int Size() const; + std::vector Batches(int batch_size) const; + void ShuffleData(Shuffle& rnd = GlobalShuffle()); + Data GetData() const; + +private: + static Shuffle& GlobalShuffle(); + Data GetBatch(Index begin, int size) const; + Data data_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/layer.cpp b/src/layer.cpp index 28ce4ec..74e7599 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,2 +1,98 @@ #include "layer.h" -namespace network {} \ No newline at end of file +#include "activation_function.h" +#include +namespace network { +namespace details { + +Random::Random(int seed) : generator_(seed) { +} +// кажется ты говорил что лучше заполнять нормальным распределением, а не равномерным +Matrix Random::NormalMatrix(Index rows, Index cols, double mean, double stdev) { + return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); +} +Vector Random::NormalVector(Index rows, double mean, double stdev) { + return Eigen::Rand::normal(rows, 1, generator_, mean, stdev); +} + +} // namespace details + +Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd) + : weights_( + InitializedWeights(static_cast(output_size), static_cast(input_size), rnd)), + bias_(InitializedBias(static_cast(output_size), rnd)), + func_(name) { +} +Layer::Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) + : weights_(weights), bias_(bias), func_(name) { +} +Matrix Layer::ApplyLinear(const Matrix& input_batch) const { + assert(weights_.cols() == input_batch.rows() && + "can not multiply input_batch on weights matrix"); + Matrix output_batch = weights_ * input_batch; + output_batch.colwise() += bias_; + return output_batch; +} + +// Vector Layer::Forward(const Vector& input) { +// assert(input.size() == weights_.cols() && "wrong size of layer"); +// Vector output = weights_ * input + bias_; +// output = func_.Activate(output); +// return output; +// } +Matrix Layer::Forward(const Matrix& input) const { + return func_.Apply(ApplyLinear(input)); +} + +void Layer::UpdateWeights(const Matrix& gradient, double learning_rate) { + assert(gradient.rows() == weights_.rows() && gradient.cols() == weights_.cols() && + "wrong size of gradient for weights"); + weights_ = weights_ - learning_rate * gradient; +} +void Layer::UpdateBias(const Vector& gradient, double learning_rate) { + assert(gradient.size() == bias_.size() && "wrong size of gradient for bias"); + bias_ = bias_ - learning_rate * gradient; +} + +Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const { + assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); + assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); + assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); + Matrix applied_linear = ApplyLinear(input_batch); + Matrix tmp = gradient; + for (int i = 0; i < gradient.cols(); ++i) { + Matrix act_func_differ = func_.GetDifferential(applied_linear.col(i)); + tmp.row(i) = tmp.row(i) * act_func_differ; + } + Matrix new_gradient(input_batch.cols(), input_batch.rows()); + new_gradient = tmp * weights_; + return new_gradient; +} + +WeightsBiasGradient Layer::GetWeightsBiasGradient(const Matrix& input_batch, + const Matrix& gradient) const { + assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); + assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); + assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); + WeightsBiasGradient grad; + Matrix applied_linear = ApplyLinear(input_batch); + Matrix matrix_grad_biases(bias_.rows(), input_batch.cols()); + for (int i = 0; i < input_batch.cols(); ++i) { + matrix_grad_biases.col(i) = + (gradient.row(i) * func_.GetDifferential(applied_linear.col(i))).transpose(); + } + grad.bias = matrix_grad_biases.colwise().sum() / matrix_grad_biases.cols(); + grad.weights = matrix_grad_biases * input_batch.transpose() / matrix_grad_biases.cols(); + return grad; +} + +Matrix Layer::InitializedWeights(Index rows, Index cols, Rand& rnd) { + return rnd.NormalMatrix(rows, cols); +} +Vector Layer::InitializedBias(Index rows, Rand& rnd) { + return rnd.NormalVector(rows); +} +Layer::Rand& Layer::GlobalRandom() { + static Rand rnd; + return rnd; +} +} // namespace network \ No newline at end of file diff --git a/src/layer.h b/src/layer.h index a204d65..0dbc207 100644 --- a/src/layer.h +++ b/src/layer.h @@ -1,24 +1,62 @@ + +#pragma once #include "activation_function.h" #include "linalg.h" namespace network { +struct WeightsBiasGradient { + Matrix weights; + Vector bias; +}; +namespace details { + +class Random { + using Generator = Eigen::Rand::Vmt19937_64; + +public: + Random(int seed = kDefaultSeed); + // лучше нормальное или равномерное? + Matrix NormalMatrix(Index rows, Index cols, double mean = 0, double stdev = 1); + Vector NormalVector(Index rows, double mean = 0, double stdev = 1); + +private: + static constexpr int kDefaultSeed = 42; + Generator generator_{kDefaultSeed}; +}; +} // namespace details + +enum class In : Index; +enum class Out : Index; class Layer { + using Rand = details::Random; + public: - void Linear(int input_size, int output_size); - void SetActivationFuntion(NamesActivationFunc name); - Vector Forward(const Vector& input); - Matrix Forward(const Matrix& input); - Vector Backward(const Vector& gradient); - Matrix Backward(const Matrix& gradient); + Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd = GlobalRandom()); + // этот конструктор больше нужен для тестирование + Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); + + Matrix ApplyLinear(const Matrix& input) const; + // Vector Forward(const Vector& input); + Matrix Forward(const Matrix& input) const; + // Vector Backward(const Vector& gradient); + Matrix Backward(const Matrix& input_batch, const Matrix& gradient) const; + + // наверно это плохая идея, но я думал так, чтобы не пересчитывать матрицу градиентов по b; Эта + // структора сделана лишь с этой целью. + WeightsBiasGradient GetWeightsBiasGradient(const Matrix& input_batch, + const Matrix& gradient) const; + void UpdateWeights(const Matrix& gradient, double learning_rate); + void UpdateBias(const Vector& gradient, double learning_rate); private: + static Rand& GlobalRandom(); + Matrix InitializedWeights(Index rows, Index cols, Rand& rnd); + Vector InitializedBias(Index rows, Rand& rnd); ActivationFunc func_; - Matrix weigts_; + Matrix weights_; Vector bias_; - bool is_set_func_; - void SetRandomWeigts(); }; } // namespace network \ No newline at end of file diff --git a/src/linalg.h b/src/linalg.h index 9980684..158babb 100644 --- a/src/linalg.h +++ b/src/linalg.h @@ -1,3 +1,4 @@ +#pragma once #include #include @@ -5,5 +6,6 @@ namespace network { using Matrix = Eigen::MatrixXd; using Vector = Eigen::VectorXd; using VectorT = Eigen::RowVectorXd; +using Index = Eigen::Index; } // namespace network \ No newline at end of file diff --git a/src/loss_function.cpp b/src/loss_function.cpp index a862340..8dbb69c 100644 --- a/src/loss_function.cpp +++ b/src/loss_function.cpp @@ -1,38 +1,44 @@ #include "loss_function.h" namespace network { +namespace details { -double Mse::GetValue(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "Mse GetValue"); - return (y_out - y_expected).array().square().mean(); -} -Vector Mse::GetGradient(const Vector &y_out, const Vector &y_expected) { +struct Mse { - assert(y_out.size() == y_expected.size() && "Mse GetGradient"); - return (2.0 / y_out.size()) * (y_out - y_expected); -} -double CrossEntropy::GetValue(const Vector &y_out, const Vector &y_expected) { + static double GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).squaredNorm(); + } + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return 2.0 * (y_out - y_expected); + } +}; +struct CrossEntropy { + static constexpr double kEPS = 1e-12; + static double GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); - assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); - const double eps = 1e-12; - return -(y_expected.array() * (y_out.array() + eps).log()).sum(); -} -Vector CrossEntropy::GetGradient(const Vector &y_out, const Vector &y_expected) { + return -(y_expected.array() * (y_out.array() + kEPS).log()).sum(); + } + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); - assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); - const double eps = 1e-12; - return -(y_expected.array() / (y_out.array() + eps)); -} -LossFunc::LossFunc(NamesLossFunctions name) { + return -(y_expected.array() / (y_out.array() + kEPS)); + } +}; + +} // namespace details +LossFunc::LossFunc(Name name) { switch (name) { - case NamesLossFunctions::Mse: - loss_func_ = Mse::GetValue; - get_grad_ = Mse::GetGradient; - case network::NamesLossFunctions::CrossEntorpy: - loss_func_ = CrossEntropy::GetValue; - get_grad_ = CrossEntropy::GetGradient; + case Name::Mse: + loss_func_ = details::Mse::GetValue; + get_grad_ = details::Mse::GetGradient; + case Name::CrossEntropy: + loss_func_ = details::CrossEntropy::GetValue; + get_grad_ = details::CrossEntropy::GetGradient; default: - assert("Problen in LossFunc constructor"); + assert(false && "Problen in LossFunc constructor"); } } double LossFunc::Dist(const Vector &y_out, const Vector &y_expected) { @@ -46,7 +52,7 @@ Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) { "Matrices have different sizes. GetGradient."); Matrix res(y_out.rows(), y_out.cols()); for (int i = 0; i < y_out.cols(); ++i) { - VectorT grad_of_one_vector = res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); + res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); } return res; } diff --git a/src/loss_function.h b/src/loss_function.h index 6247c97..f57074c 100644 --- a/src/loss_function.h +++ b/src/loss_function.h @@ -3,27 +3,17 @@ #include namespace network { -enum class NamesLossFunctions { Mse, CrossEntorpy }; - -struct Mse { - static double GetValue(const Vector &y_out, const Vector &y_expected); - static Vector GetGradient(const Vector &y_out, const Vector &y_expected); -}; -struct CrossEntropy { - static double GetValue(const Vector &y_out, const Vector &y_expected); - static Vector GetGradient(const Vector &y_out, const Vector &y_expected); -}; - class LossFunc { public: - LossFunc(NamesLossFunctions name); + enum class Name { Mse, CrossEntropy }; + LossFunc(Name name); double Dist(const Vector &y_out, const Vector &y_expected); VectorT GetGradient(const Vector &y_out, const Vector &y_expected); Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); private: - std::function loss_func_; - std::function get_grad_; + std::function loss_func_; + std::function get_grad_; }; } // namespace network \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index c6299ac..951f7d6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,17 +1,8 @@ -#include -#include -#include +#include "adam_optimizer.h" #include -using RandGen = Eigen::Rand::Vmt19937_64; -RandGen& GetRng() { - static RandGen rng = 1; - return rng; -} +// main я пока использую, чтобы посмотреть так все работает как я ожидаю или нет. В итоговой проект +// он не войдет. + int main() { - int rows = 2; - int cols = 2; - Eigen::MatrixXd result = Eigen::Rand::normal(rows, cols, GetRng()); - std::cout << result << std::endl; - return 0; } \ No newline at end of file diff --git a/src/net.cpp b/src/net.cpp index e69de29..881bdd1 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -0,0 +1,23 @@ +#include "net.h" +#include "activation_function.h" + +// НЕ ЧИТАЙ +namespace network { +Net::Net(DataLoader&& dl, LossFunc::Name name) : dl_(std::move(dl)), loss_func_(name) { +} +void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { + layers_.emplace_back(Layer(input_size, output_size, name)); +} +std::vector Net::Forward(const Matrix& batch) { + Matrix cur_output = batch; + std::vector output_layers; + output_layers.push_back(cur_output); + for (const Layer& layer : layers_) { + cur_output = layer.Forward(cur_output); + output_layers.push_back(cur_output); + } + return output_layers; +} +void Backward() { +} +} // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h index e69de29..166e646 100644 --- a/src/net.h +++ b/src/net.h @@ -0,0 +1,24 @@ +#pragma once +#include "activation_function.h" +#include "linalg.h" +#include "layer.h" +#include "loss_function.h" +#include "dataloader.h" +#include + +namespace network { + +/// НЕ ЧИТАЙ +class Net { + Net(DataLoader&& dl, LossFunc::Name name); + void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); + std::vector Forward(const Matrix& batch); + void Backward(); + double RunTest(); + +private: + DataLoader dl_; + LossFunc loss_func_; + std::vector layers_; +}; +} // namespace network \ No newline at end of file diff --git a/tests/test_activation_function.cpp b/tests/test_activation_function.cpp index c53dfc3..5fa37d6 100644 --- a/tests/test_activation_function.cpp +++ b/tests/test_activation_function.cpp @@ -1,23 +1,22 @@ #include +#include "linalg.h" #include "activation_function.h" - - - - - -TEST(ActivationfuncTest, Relu){ - network::ActivationFunc func(network::NamesActivationFunc::ReLU); - Eigen::VectorXd vector(2); - vector[1] = 1; - vector[0] = 1; - Eigen::MatrixXd g = func.Activate(vector); - EXPECT_EQ(vector, g); - +namespace network { + +TEST(ActivationfuncTest, Relu) { + ActivationFunc act(ActivationFunc::Name::ReLU); + Matrix a(2, 2); + a << 1, -1, 2, -2; + Matrix expected_apply(2, 2); + expected_apply << 1, 0, 2, 0; + EXPECT_EQ(expected_apply, act.Apply(a)); + Matrix expected_diff(2, 2); + expected_diff << 1, 0, 0, 1; + EXPECT_EQ(act.GetDifferential(a.col(0)), expected_diff); } - - -int main(int argc, char **argv) { +} // namespace network +int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); -} \ No newline at end of file +} \ No newline at end of file diff --git a/tests/test_dataloader.cpp b/tests/test_dataloader.cpp new file mode 100644 index 0000000..e210121 --- /dev/null +++ b/tests/test_dataloader.cpp @@ -0,0 +1,18 @@ +#include +#include "dataloader.h" + + +TEST(DataLoder, Load){ + + + + +} + + + + +int main(int argc, char** argv){ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/tests/test_layer.cpp b/tests/test_layer.cpp index e69de29..c27acec 100644 --- a/tests/test_layer.cpp +++ b/tests/test_layer.cpp @@ -0,0 +1,57 @@ +#include +#include "activation_function.h" +#include "linalg.h" +#include "layer.h" + +namespace network { +namespace tests_layer { + +Matrix weights_matrix(3, 2); +Vector bias(3); +Matrix input(2, 3); +Matrix expected_linear_output(3, 3); +Matrix expected_forward_matrix(3, 3); +Matrix expected_backward_matrix(3, 2); +Vector expected_grad_b(3); +Matrix expected_grad_a(3, 2); +Matrix gradient(3, 3); +void SetTestParamers() { + weights_matrix << 6, 1, 1, 1, 10, 7; + bias << 1, 1, 1; + input << 1, -2, 3, -4, 5, -5; + expected_linear_output << 3, -6, 14, -2, 4, -1, -17, 16, -4; + expected_forward_matrix << 3, 0, 14, 0, 4, 0, 0, 16, 0; + gradient << 8, 10, 1, 7, 6, 6, -9, -5, 1; + ///////////// + expected_backward_matrix << 48, 8, 66, 48, -54, -9; + expected_grad_b << -1, 6, 6; + expected_grad_a << -19, 13, -12, 30, -12, 30; + expected_grad_b /= 3; + expected_grad_a /= 3; +} + +TEST(Correction, Forward) { + Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); + Matrix output = layer.ApplyLinear(input); + EXPECT_EQ(output, expected_linear_output); + Matrix forward_matrix = layer.Forward(input); + EXPECT_EQ(forward_matrix, expected_forward_matrix); +} +TEST(Correction, Backward) { + Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); + Matrix bacward_matrix = layer.Backward(input, gradient); + EXPECT_EQ(expected_backward_matrix, bacward_matrix); +} +TEST(Correction, Gradients) { + Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); + WeightsBiasGradient grad_weights_bias = layer.GetWeightsBiasGradient(input, gradient); + EXPECT_EQ(expected_grad_a, grad_weights_bias.weights); + EXPECT_EQ(expected_grad_b, grad_weights_bias.bias); +} +// namespace network +} // namespace tests_layer +} // namespace network +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/test_loss_function.cpp b/tests/test_loss_function.cpp index e69de29..7e294ad 100644 --- a/tests/test_loss_function.cpp +++ b/tests/test_loss_function.cpp @@ -0,0 +1,2 @@ +#include +#include "linalg.h" \ No newline at end of file From 8e1823ebaed87d328da7b2c369a835a446130a2f Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Sat, 22 Mar 2025 20:44:34 +0300 Subject: [PATCH 05/13] added class net --- CMakeLists.txt | 2 +- src/activation_function.cpp | 1 + src/adam_optimizer.cpp | 11 +++--- src/adam_optimizer.h | 7 ++-- src/dataloader.cpp | 2 -- src/layer.cpp | 24 ++++++++----- src/layer.h | 6 ++-- src/loss_function.cpp | 17 ++++++---- src/loss_function.h | 2 +- src/main.cpp | 36 ++++++++++++++++++-- src/net.cpp | 67 +++++++++++++++++++++++++++++++------ src/net.h | 30 +++++++++++++---- 12 files changed, 158 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b15d913..a21cce8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ add_executable(test_layer src/activation_function.cpp) target_link_libraries(test_layer PRIVATE gtest gtest_main) -add_executable(main src/main.cpp src/adam_optimizer.cpp) +add_executable(main src/main.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index 5233528..4a307ab 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -71,6 +71,7 @@ void ActivationFunc::SetFunction(ActivationFunc::Name name) { case ActivationFunc::Name::Softmax: apply_ = details::Softmax::Apply; differential_ = details::Softmax::GetDifferential; + break; default: assert(false && "invalid arguments in ActivationFunc constructor"); } diff --git a/src/adam_optimizer.cpp b/src/adam_optimizer.cpp index 1bde409..df86f0a 100644 --- a/src/adam_optimizer.cpp +++ b/src/adam_optimizer.cpp @@ -3,18 +3,19 @@ namespace network { -AdamOptimizer::AdamOptimizer(const Matrix& parametr, double start_learning_rate, double beta1, +AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, double start_learning_rate, double beta1, double beta2) : t_(0), learning_rate_(start_learning_rate), beta1_(beta1), beta2_(beta2), - m_t_(Matrix::Zero(parametr.rows(), parametr.cols())), - v_t_(Matrix::Zero(parametr.rows(), parametr.cols())) { + m_t_(Matrix::Zero(static_cast(rows), static_cast(cols))), + v_t_(Matrix::Zero(static_cast(rows), static_cast(cols))) { } -Matrix AdamOptimizer::ComputeUpdate(const Matrix& gradient) { - assert(gradient.rows() == m_t_.rows() && gradient.cols() && m_t_.cols() && "invalid gradient"); +Matrix AdamOptimizer::ComputeCorrection(const Matrix& gradient) { + assert(gradient.rows() == m_t_.rows() && "invalid gradient"); + assert(gradient.cols() == m_t_.cols() && "invalid gradient"); t_ += 1; m_t_ = beta1_ * m_t_ + (1 - beta1_) * gradient; Matrix squared_gradient = gradient.cwiseProduct(gradient); diff --git a/src/adam_optimizer.h b/src/adam_optimizer.h index b9efd00..3335809 100644 --- a/src/adam_optimizer.h +++ b/src/adam_optimizer.h @@ -2,11 +2,14 @@ #include "linalg.h" namespace network { +enum class Rows : Index; +enum class Cols : Index; + class AdamOptimizer { public: - AdamOptimizer(const Matrix& parametr, double start_learning_rate = kDefaultStartLearningRate, + AdamOptimizer(Rows rows, Cols cols, double start_learning_rate = kDefaultStartLearningRate, double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); - Matrix ComputeUpdate(const Matrix& gradient); + Matrix ComputeCorrection(const Matrix& gradient); private: static constexpr double kDefaultStartLearningRate = 3e-4; diff --git a/src/dataloader.cpp b/src/dataloader.cpp index 67f131d..36ff139 100644 --- a/src/dataloader.cpp +++ b/src/dataloader.cpp @@ -29,13 +29,11 @@ void Shuffle::ShuffleData(Index begin, Index end, Data& data) { DataLoader::DataLoader(Data&& data) { assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); data_ = std::move(data); - ShuffleData(); } DataLoader::DataLoader(const Data& data) { assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); data_ = data; - ShuffleData(); } int DataLoader::Size() const { return data_.input.cols(); diff --git a/src/layer.cpp b/src/layer.cpp index 74e7599..56db892 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -43,14 +43,14 @@ Matrix Layer::Forward(const Matrix& input) const { return func_.Apply(ApplyLinear(input)); } -void Layer::UpdateWeights(const Matrix& gradient, double learning_rate) { - assert(gradient.rows() == weights_.rows() && gradient.cols() == weights_.cols() && - "wrong size of gradient for weights"); - weights_ = weights_ - learning_rate * gradient; +void Layer::UpdateWeights(const Matrix& correction) { + assert(weights_.cols() == correction.cols() && weights_.rows() == correction.rows() && + "invalid correction"); + weights_ -= correction; } -void Layer::UpdateBias(const Vector& gradient, double learning_rate) { - assert(gradient.size() == bias_.size() && "wrong size of gradient for bias"); - bias_ = bias_ - learning_rate * gradient; +void Layer::UpdateBias(const Vector correction) { + assert(correction.rows() == bias_.rows() && "invalid correction"); + bias_ -= correction; } Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const { @@ -59,7 +59,7 @@ Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); Matrix applied_linear = ApplyLinear(input_batch); Matrix tmp = gradient; - for (int i = 0; i < gradient.cols(); ++i) { + for (int i = 0; i < tmp.rows(); ++i) { Matrix act_func_differ = func_.GetDifferential(applied_linear.col(i)); tmp.row(i) = tmp.row(i) * act_func_differ; } @@ -80,10 +80,16 @@ WeightsBiasGradient Layer::GetWeightsBiasGradient(const Matrix& input_batch, matrix_grad_biases.col(i) = (gradient.row(i) * func_.GetDifferential(applied_linear.col(i))).transpose(); } - grad.bias = matrix_grad_biases.colwise().sum() / matrix_grad_biases.cols(); + grad.bias = matrix_grad_biases.rowwise().sum() / matrix_grad_biases.cols(); grad.weights = matrix_grad_biases * input_batch.transpose() / matrix_grad_biases.cols(); return grad; } +Index Layer::GetWeightCols() const { + return weights_.cols(); +} +Index Layer::GetWeightRows() const { + return weights_.rows(); +} Matrix Layer::InitializedWeights(Index rows, Index cols, Rand& rnd) { return rnd.NormalMatrix(rows, cols); diff --git a/src/layer.h b/src/layer.h index 0dbc207..ab06a9e 100644 --- a/src/layer.h +++ b/src/layer.h @@ -47,8 +47,10 @@ class Layer { // структора сделана лишь с этой целью. WeightsBiasGradient GetWeightsBiasGradient(const Matrix& input_batch, const Matrix& gradient) const; - void UpdateWeights(const Matrix& gradient, double learning_rate); - void UpdateBias(const Vector& gradient, double learning_rate); + void UpdateWeights(const Matrix& correction); + void UpdateBias(const Vector correction); + Index GetWeightCols() const; + Index GetWeightRows() const; private: static Rand& GlobalRandom(); diff --git a/src/loss_function.cpp b/src/loss_function.cpp index 8dbb69c..de90785 100644 --- a/src/loss_function.cpp +++ b/src/loss_function.cpp @@ -34,9 +34,11 @@ LossFunc::LossFunc(Name name) { case Name::Mse: loss_func_ = details::Mse::GetValue; get_grad_ = details::Mse::GetGradient; + break; case Name::CrossEntropy: loss_func_ = details::CrossEntropy::GetValue; get_grad_ = details::CrossEntropy::GetGradient; + break; default: assert(false && "Problen in LossFunc constructor"); } @@ -44,17 +46,18 @@ LossFunc::LossFunc(Name name) { double LossFunc::Dist(const Vector &y_out, const Vector &y_expected) { return loss_func_(y_out, y_expected); } -VectorT LossFunc::GetGradient(const Vector &y_out, const Vector &y_expected) { - return get_grad_(y_out, y_expected).transpose(); -} -Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) { + +Matrix LossFunc::GetGradient(const Matrix &y_out, const Matrix &y_expected) { assert(y_out.cols() == y_expected.cols() && y_out.rows() == y_expected.rows() && "Matrices have different sizes. GetGradient."); - Matrix res(y_out.rows(), y_out.cols()); - for (int i = 0; i < y_out.cols(); ++i) { - res.row(i) = GetGradient(y_out.col(i), y_expected.col(i)); + Matrix res(y_out.cols(), y_out.rows()); + for (int i = 0; i < res.rows(); ++i) { + res.row(i) = GetVectorGrad(y_out.col(i), y_expected.col(i)); } return res; } +VectorT LossFunc::GetVectorGrad(const Vector &y_out, const Vector &y_expected) { + return get_grad_(y_out, y_expected).transpose(); +} } // namespace network diff --git a/src/loss_function.h b/src/loss_function.h index f57074c..f795167 100644 --- a/src/loss_function.h +++ b/src/loss_function.h @@ -8,10 +8,10 @@ class LossFunc { enum class Name { Mse, CrossEntropy }; LossFunc(Name name); double Dist(const Vector &y_out, const Vector &y_expected); - VectorT GetGradient(const Vector &y_out, const Vector &y_expected); Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); private: + VectorT GetVectorGrad(const Vector &y_out, const Vector &y_expected); std::function loss_func_; std::function get_grad_; }; diff --git a/src/main.cpp b/src/main.cpp index 951f7d6..7c5996b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,8 +1,40 @@ -#include "adam_optimizer.h" +#include "activation_function.h" +#include "loss_function.h" +#include "net.h" #include +#include // main я пока использую, чтобы посмотреть так все работает как я ожидаю или нет. В итоговой проект // он не войдет. +using namespace network; +void RandomMatrix(Matrix& mat) { + for (int i = 0; i < mat.cols(); ++i) { + for (int j = 0; j < mat.rows(); ++j) { + std::mt19937 gen(i + j); + std::uniform_real_distribution dist(0, 100); + mat(j, i) = dist(gen); + } + } +} int main() { -} \ No newline at end of file + + Matrix input(784, 60000); + Matrix output(9, 60000); + RandomMatrix(input); + RandomMatrix(output); + + Data data{input, output}; + DataLoader dl(std::move(data)); + // std::vector dt = dl.Batches(1); + // dl.ShuffleData(); + // dt = dl.Batches(2); + + Net nn(std::move(dl), LossFunc::Name::CrossEntropy); + nn.AddLayer(In{784}, Out{3}, ActivationFunc::Name::ReLU); + nn.AddLayer(In{3}, Out{7}, ActivationFunc::Name::Sigmoid); + nn.AddLayer(In{7}, Out{9}, ActivationFunc::Name::Softmax); + nn.Backward(60000, 1); + + // nn.Backward(2, 100); +} diff --git a/src/net.cpp b/src/net.cpp index 881bdd1..84ebef5 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,23 +1,70 @@ #include "net.h" +#include #include "activation_function.h" +#include "adam_optimizer.h" +#include "layer.h" -// НЕ ЧИТАЙ namespace network { Net::Net(DataLoader&& dl, LossFunc::Name name) : dl_(std::move(dl)), loss_func_(name) { } void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { - layers_.emplace_back(Layer(input_size, output_size, name)); + assert((layers_.empty() || layers_.back().GetWeightRows() == static_cast(input_size)) && + "incorrect layer"); + layers_.emplace_back(In{input_size}, Out{output_size}, name); } -std::vector Net::Forward(const Matrix& batch) { - Matrix cur_output = batch; - std::vector output_layers; - output_layers.push_back(cur_output); +Net::ComputedBatches Net::Forward(const Matrix& batch) const { + Matrix cur_input = batch; + std::vector inputs; + inputs.push_back(cur_input); for (const Layer& layer : layers_) { - cur_output = layer.Forward(cur_output); - output_layers.push_back(cur_output); + cur_input = layer.Forward(cur_input); + inputs.push_back(cur_input); } - return output_layers; + return inputs; } -void Backward() { + +Matrix Net::Evalute(const Matrix& batch) const { + Matrix cur_batch = batch; + for (const Layer& layer : layers_) { + cur_batch = layer.Forward(cur_batch); + } + return cur_batch; } + +void Net::Backward(int batch_size, int num_epochs, double start_learning_rate, double beta1, + double beta2) { + assert(!layers_.empty() && "empty layers"); + for (int i = 0; i < layers_.size(); ++i) { + Index rows = layers_[i].GetWeightRows(); + Index cols = layers_[i].GetWeightCols(); + AdamOptimizer opt_of_weights(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); + AdamOptimizer opt_of_bias(Rows{rows}, Cols{1}, start_learning_rate, beta1, beta2); + opts_.push_back({opt_of_weights, opt_of_bias}); + } + for (int epoch = 0; epoch < num_epochs; ++epoch) { + dl_.ShuffleData(); + for (const Data& train_batch : dl_.Batches(batch_size)) { + TrainBatch(train_batch); + } + } +} + +void Net::TrainBatch(const Data& data) { + assert(!layers_.empty() && "no layers"); + assert(layers_.size() == opts_.size() && "different sizes of optimizers and layers"); + ComputedBatches computed_batces = Forward(data.input); + assert(computed_batces.size() - layers_.size() == 1 && + "invalide size of layers_ or computed_batches"); + Matrix cur_gradient = loss_func_.GetGradient(computed_batces.back(), data.output); + for (int i = layers_.size() - 1; i >= 0; --i) { + WeightsBiasGradient grad = + layers_[i].GetWeightsBiasGradient(computed_batces[i], cur_gradient); + Matrix correction_weights = opts_[i].weights.ComputeCorrection(grad.weights); + Vector correction_bias = opts_[i].bias.ComputeCorrection(grad.bias); + cur_gradient = layers_[i].Backward(computed_batces[i], cur_gradient); + layers_[i].UpdateWeights(correction_weights); + layers_[i].UpdateBias(correction_bias); + } +} + } // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h index 166e646..341066a 100644 --- a/src/net.h +++ b/src/net.h @@ -1,24 +1,42 @@ #pragma once -#include "activation_function.h" #include "linalg.h" +#include "activation_function.h" #include "layer.h" #include "loss_function.h" #include "dataloader.h" +#include "adam_optimizer.h" #include namespace network { +namespace details { -/// НЕ ЧИТАЙ +struct OptimizersParams { + AdamOptimizer weights; + AdamOptimizer bias; +}; +} // namespace details class Net { + using Layers = std::vector; + using ComputedBatches = std::vector; + using Optimizers = std::vector; + +public: Net(DataLoader&& dl, LossFunc::Name name); void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); - std::vector Forward(const Matrix& batch); - void Backward(); - double RunTest(); + ComputedBatches Forward(const Matrix& batch) const; + Matrix Evalute(const Matrix& batch) const; + void Backward(int batch_size, int num_epochs, + double start_learning_rate = kDefaultStartLearningRate, + double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); private: + void TrainBatch(const Data& data); + static constexpr double kDefaultStartLearningRate = 3e-4; + static constexpr double kDefaultBeta1 = 0.9; + static constexpr double kDefaultBeta2 = 0.99; DataLoader dl_; LossFunc loss_func_; - std::vector layers_; + Layers layers_; + Optimizers opts_; }; } // namespace network \ No newline at end of file From 118b50ba83c6bddd79d54b671156017e3ba65fef Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Sun, 23 Mar 2025 01:24:19 +0300 Subject: [PATCH 06/13] test mnist in development... --- .gitmodules | 3 +++ libs/mnist | 1 + 2 files changed, 4 insertions(+) create mode 160000 libs/mnist diff --git a/.gitmodules b/.gitmodules index 3ec7fd0..c7531cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "libs/googletest"] path = libs/googletest url = https://github.com/google/googletest.git +[submodule "libs/mnist"] + path = libs/mnist + url = https://github.com/wichtounet/mnist.git diff --git a/libs/mnist b/libs/mnist new file mode 160000 index 0000000..3b65c35 --- /dev/null +++ b/libs/mnist @@ -0,0 +1 @@ +Subproject commit 3b65c35ede53b687376c4302eeb44fdf76e0129b From 419932a68773d7347e86523b6d91be0898b32f67 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Sun, 23 Mar 2025 01:25:19 +0300 Subject: [PATCH 07/13] test mnist in development... --- CMakeLists.txt | 5 ++- src/main.cpp | 82 +++++++++++++++++++++++++++++++++++--------------- src/net.cpp | 3 +- src/net.h | 1 + 4 files changed, 64 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a21cce8..c9820c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,11 +11,13 @@ include_directories( ${CMAKE_SOURCE_DIR}/libs/eigen ${CMAKE_SOURCE_DIR}/libs/EigenRand ${CMAKE_SOURCE_DIR}/libs/googletest/googletest/include - ${CMAKE_SOURCE_DIR}/src + ${CMAKE_SOURCE_DIR}/libs/mnist/include + ${CMAKE_SOURCE_DIR}/src/include ) add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) +# add_subdirectory(libs/mnist) # add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) # add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) @@ -42,6 +44,7 @@ add_executable(main src/main.cpp src/adam_optimizer.cpp src/net.cpp src/loss_fun + enable_testing() add_test(NAME test_activation_function COMMAND test_activation_function) add_test(NAME test_loss_function COMMAND test_loss_function) diff --git a/src/main.cpp b/src/main.cpp index 7c5996b..f36b97b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,40 +1,72 @@ -#include "activation_function.h" + #include "loss_function.h" #include "net.h" #include #include +#include +#include "mnist/mnist_reader.hpp" -// main я пока использую, чтобы посмотреть так все работает как я ожидаю или нет. В итоговой проект -// он не войдет. +//за это сори, я по фасту накидывал, чтобы протестить using namespace network; +Matrix ConvertToMatrix(const std::vector>& images) { + size_t num_images = images.size(); + Matrix eigen_images(784, num_images); -void RandomMatrix(Matrix& mat) { - for (int i = 0; i < mat.cols(); ++i) { - for (int j = 0; j < mat.rows(); ++j) { - std::mt19937 gen(i + j); - std::uniform_real_distribution dist(0, 100); - mat(j, i) = dist(gen); + for (size_t i = 0; i < num_images; ++i) { + for (int j = 0; j < 784; ++j) { + eigen_images(j, i) = static_cast(images[i][j]) / 255.0; } } + + return eigen_images; } -int main() { +Matrix LabelToVector(const std::vector& labels) { + size_t num_labels = labels.size(); + Matrix onehot(10, num_labels); + onehot.setZero(); - Matrix input(784, 60000); - Matrix output(9, 60000); - RandomMatrix(input); - RandomMatrix(output); + for (size_t i = 0; i < num_labels; ++i) { + onehot(labels[i], i) = 1.0; + } - Data data{input, output}; - DataLoader dl(std::move(data)); - // std::vector dt = dl.Batches(1); - // dl.ShuffleData(); - // dt = dl.Batches(2); + return onehot; +} - Net nn(std::move(dl), LossFunc::Name::CrossEntropy); - nn.AddLayer(In{784}, Out{3}, ActivationFunc::Name::ReLU); - nn.AddLayer(In{3}, Out{7}, ActivationFunc::Name::Sigmoid); - nn.AddLayer(In{7}, Out{9}, ActivationFunc::Name::Softmax); - nn.Backward(60000, 1); +std::pair GetTrainDataSet(std::string path_to_data) { + auto dataset = mnist::read_dataset(path_to_data); + assert(!dataset.training_images.empty() && "empty dataset"); + Matrix train_images = ConvertToMatrix(dataset.training_images); + Matrix train_labels = LabelToVector(dataset.training_labels); + Data train{train_images, train_labels}; + Matrix test_images = ConvertToMatrix(dataset.test_images); + Matrix test_labels = LabelToVector(dataset.test_labels); + Data test_data{test_images, test_labels}; + return std::make_pair(train, test_data); +} - // nn.Backward(2, 100); +int main() { + std::string path = "../libs/mnist"; + std::pair ans = GetTrainDataSet(path); + DataLoader dl(std::move(ans.first)); + Net nn(std::move(dl), LossFunc::Name::CrossEntropy); + nn.AddLayer(In{784}, Out{256}, ActivationFunc::Name::ReLU); + nn.AddLayer(In{256}, Out{10}, ActivationFunc::Name::Softmax); + nn.Backward(64, 15); + std::cout << "///////////////////////////////// TEST PATR//////////////////////////////////////" + << std::endl; + Matrix evalute_test = nn.Evalute(ans.second.input); + int size_test = ans.second.input.cols(); + int cnt = 0; + std::cout << " Size TESt " << size_test << std::endl; + Index max_our_ind; + Index max_test_ind; + for (int i = 0; i < size_test; ++i) { + evalute_test.col(i).maxCoeff(&max_our_ind); + ans.second.output.col(i).maxCoeff(&max_test_ind); + if (max_test_ind == max_our_ind) { + ++cnt; + } + } + double accuracy = (static_cast(cnt) / size_test) * 100; + std::cout<< accuracy << std::endl; } diff --git a/src/net.cpp b/src/net.cpp index 84ebef5..79d6462 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -3,7 +3,7 @@ #include "activation_function.h" #include "adam_optimizer.h" #include "layer.h" - +#include namespace network { Net::Net(DataLoader&& dl, LossFunc::Name name) : dl_(std::move(dl)), loss_func_(name) { } @@ -35,6 +35,7 @@ void Net::Backward(int batch_size, int num_epochs, double start_learning_rate, d double beta2) { assert(!layers_.empty() && "empty layers"); for (int i = 0; i < layers_.size(); ++i) { + Index rows = layers_[i].GetWeightRows(); Index cols = layers_[i].GetWeightCols(); AdamOptimizer opt_of_weights(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); diff --git a/src/net.h b/src/net.h index 341066a..8b33024 100644 --- a/src/net.h +++ b/src/net.h @@ -15,6 +15,7 @@ struct OptimizersParams { AdamOptimizer bias; }; } // namespace details + class Net { using Layers = std::vector; using ComputedBatches = std::vector; From 26792ffd88085c471ad3b50322d707885c9d1594 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Wed, 26 Mar 2025 03:11:01 +0300 Subject: [PATCH 08/13] fixed some bugs --- .gitignore | 2 ++ CMakeLists.txt | 3 +- src/activation_function.cpp | 56 ++++++++++++++++++++------------ src/activation_function.h | 9 ++++-- src/adam_optimizer.cpp | 33 +++++++++++-------- src/adam_optimizer.h | 12 +++++-- src/dataloader.cpp | 9 +++++- src/layer.cpp | 45 +++++++++++++++----------- src/layer.h | 18 ++++------- src/main.cpp | 24 ++++++++------ src/net.cpp | 64 +++++++++++++++++++++---------------- src/net.h | 17 +++++----- 12 files changed, 175 insertions(+), 117 deletions(-) diff --git a/.gitignore b/.gitignore index 483404a..3a5f27a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ compile_commands.json .vscode* build* libs/eigen +libs/tqdm.cpp +src/file_reader_writer* diff --git a/CMakeLists.txt b/CMakeLists.txt index c9820c1..e6ed887 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,12 +18,13 @@ include_directories( add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) # add_subdirectory(libs/mnist) + # add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) # add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) add_executable(test_activation_function tests/test_activation_function.cpp - src/activation_function.cpp # Добавляем исходный файл с реализацией + src/activation_function.cpp ) target_link_libraries(test_activation_function PRIVATE gtest gtest_main) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index 4a307ab..7c9280a 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -8,9 +8,9 @@ struct Sigmoid { return x.unaryExpr([](double x) { return 1.0 / (1.0 + std::exp(-x)); }); } static Matrix GetDifferential(const Vector& x) { - Vector activated_vector = Apply(x); - Vector diff = activated_vector.unaryExpr([](double x) { return (1.0 - x) * x; }); - return diff.asDiagonal(); + Vector applied_sigmoid = Apply(x); + Vector differential = applied_sigmoid.unaryExpr([](double x) { return (1.0 - x) * x; }); + return differential.asDiagonal(); } }; @@ -19,9 +19,9 @@ struct Tanh { return x.unaryExpr([](double x) { return std::tanh(x); }); } static Matrix GetDifferential(const Vector& x) { - Vector activated_vector = Apply(x); - Vector diff = activated_vector.unaryExpr([](double x) { return 1.0 - x * x; }); - return diff.asDiagonal(); + Vector applied_tanh = Apply(x); + Vector differential = applied_tanh.unaryExpr([](double x) { return 1.0 - x * x; }); + return differential.asDiagonal(); } }; @@ -30,25 +30,28 @@ struct ReLU { return x.unaryExpr([](double x) { return std::max(0.0, x); }); } static Matrix GetDifferential(const Vector& x) { - Vector diff = x.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); - return diff.asDiagonal(); + Vector differential = x.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); + return differential.asDiagonal(); } }; struct Softmax { static Vector Apply(const Vector& x) { Vector tmp = x.unaryExpr([](double x) { return std::exp(x); }); - Vector activated_vector = tmp / tmp.sum(); - return activated_vector; + Vector applied_softmax = tmp / tmp.sum(); + return applied_softmax; } static Matrix GetDifferential(const Vector& x) { - Vector tmp = Apply(x); - Matrix jacobian(tmp.size(), tmp.size()); - for (int i = 0; i < tmp.size(); ++i) { - for (int j = 0; j < tmp.size(); ++j) { - jacobian(i, j) = tmp(i) * ((i == j) ? (1.0 - tmp(j)) : -tmp(j)); - } - } - return jacobian; + Vector applied_softmax = Apply(x); + Matrix tmp = applied_softmax.asDiagonal(); + return tmp - applied_softmax * applied_softmax.transpose(); + } +}; +struct Linear { + static Vector Apply(const Vector& x) { + return x; + } + static Matrix GetDifferential(const Vector& x) { + return Eigen::MatrixXd::Identity(x.rows(), x.rows()); } }; @@ -57,21 +60,31 @@ struct Softmax { void ActivationFunc::SetFunction(ActivationFunc::Name name) { switch (name) { case ActivationFunc::Name::Sigmoid: + id_ = static_cast(ActivationFunc::Name::Sigmoid); apply_ = details::Sigmoid::Apply; differential_ = details::Sigmoid::GetDifferential; break; case ActivationFunc::Name::ReLU: + id_ = static_cast(ActivationFunc::Name::ReLU); apply_ = details::ReLU::Apply; differential_ = details::ReLU::GetDifferential; break; case ActivationFunc::Name::Tanh: + id_ = static_cast(ActivationFunc::Name::Tanh); apply_ = details::Tanh::Apply; differential_ = details::Tanh::GetDifferential; break; case ActivationFunc::Name::Softmax: + id_ = static_cast(ActivationFunc::Name::Softmax); apply_ = details::Softmax::Apply; differential_ = details::Softmax::GetDifferential; break; + case ActivationFunc::Name::Linear: + id_ = static_cast(ActivationFunc::Name::Linear); + apply_ = details::Linear::Apply; + differential_ = details::Linear::GetDifferential; + break; + default: assert(false && "invalid arguments in ActivationFunc constructor"); } @@ -80,9 +93,7 @@ void ActivationFunc::SetFunction(ActivationFunc::Name name) { ActivationFunc::ActivationFunc(ActivationFunc::Name name) { SetFunction(name); } -Vector ActivationFunc::Apply(const Vector& vector) const { - return apply_(vector); -} + Matrix ActivationFunc::Apply(const Matrix& x) const { Matrix res(x.rows(), x.cols()); @@ -95,4 +106,7 @@ Matrix ActivationFunc::Apply(const Matrix& x) const { Matrix ActivationFunc::GetDifferential(const Vector& x) const { return differential_(x); } +int ActivationFunc::GetId() const { + return id_; +} } // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h index 0dea46a..1a398ac 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -1,23 +1,26 @@ #pragma once -#include "linalg.h" #include #include +#include "linalg.h" + namespace network { class ActivationFunc { using Function = std::function; using Differential = std::function; public: - enum class Name { Sigmoid, ReLU, Tanh, Softmax }; + ActivationFunc() = default; + enum class Name { Sigmoid, ReLU, Tanh, Softmax, Linear }; explicit ActivationFunc(Name name); void SetFunction(Name name); - Vector Apply(const Vector& x) const; Matrix Apply(const Matrix& x) const; Matrix GetDifferential(const Vector& x) const; + int GetId() const; private: Function apply_; Differential differential_; + int id_; }; } // namespace network diff --git a/src/adam_optimizer.cpp b/src/adam_optimizer.cpp index df86f0a..eacb3e2 100644 --- a/src/adam_optimizer.cpp +++ b/src/adam_optimizer.cpp @@ -9,22 +9,29 @@ AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, double start_learning_rate, d learning_rate_(start_learning_rate), beta1_(beta1), beta2_(beta2), - m_t_(Matrix::Zero(static_cast(rows), static_cast(cols))), - v_t_(Matrix::Zero(static_cast(rows), static_cast(cols))) { + moments_{Matrix::Zero(static_cast(rows), static_cast(cols)), + Matrix::Zero(static_cast(rows), static_cast(cols)), + Vector::Zero(static_cast(rows)), Vector::Zero(static_cast(rows))} { } -Matrix AdamOptimizer::ComputeCorrection(const Matrix& gradient) { - assert(gradient.rows() == m_t_.rows() && "invalid gradient"); - assert(gradient.cols() == m_t_.cols() && "invalid gradient"); +void AdamOptimizer::GetCorrection(Matrix& weights_gradient, Vector& bias_gradient) { + assert(weights_gradient.rows() == moments_.weights_m_t.rows() && "invalid weights_gradient"); + assert(weights_gradient.cols() == moments_.weights_m_t.cols() && "invalid weights_gradient"); + assert(bias_gradient.size() == moments_.bias_m_t.size() && "invalid bias_gradient"); t_ += 1; - m_t_ = beta1_ * m_t_ + (1 - beta1_) * gradient; - Matrix squared_gradient = gradient.cwiseProduct(gradient); - v_t_ = beta2_ * v_t_ + (1 - beta2_) * squared_gradient; - double m_bias_correction = 1.0 - std::pow(beta1_, t_); - double v_bias_correction = 1.0 - std::pow(beta2_, t_); - Matrix m_tmp = m_t_ / m_bias_correction; - Matrix v_tmp = v_t_ / v_bias_correction; - return learning_rate_ * m_tmp.array() / (v_tmp.array().sqrt() + kEps); + moments_.weights_m_t = beta1_ * moments_.weights_m_t + (1 - beta1_) * weights_gradient; + moments_.bias_m_t = beta1_ * moments_.bias_m_t + (1 - beta1_) * bias_gradient; + + Matrix squared_weights_gradient = weights_gradient.cwiseProduct(weights_gradient); + Vector squared_bias_gradient = bias_gradient.cwiseProduct(bias_gradient); + moments_.weights_v_t = beta2_ * moments_.weights_v_t + (1 - beta2_) * squared_weights_gradient; + moments_.bias_v_t = beta2_ * moments_.bias_v_t + (1 - beta2_) * squared_bias_gradient; + double m_correction = 1.0 - std::pow(beta1_, t_); + double v_correction = 1.0 - std::pow(beta2_, t_); + weights_gradient = learning_rate_ * (moments_.weights_m_t / m_correction).array() / + ((moments_.weights_v_t / v_correction).array().sqrt() + kEps); + bias_gradient = learning_rate_ * (moments_.bias_m_t / m_correction).array() / + ((moments_.bias_v_t / v_correction).array().sqrt() + kEps); } } // namespace network diff --git a/src/adam_optimizer.h b/src/adam_optimizer.h index 3335809..308b981 100644 --- a/src/adam_optimizer.h +++ b/src/adam_optimizer.h @@ -6,10 +6,17 @@ enum class Rows : Index; enum class Cols : Index; class AdamOptimizer { + struct Moments { + Matrix weights_m_t; + Matrix weights_v_t; + Vector bias_m_t; + Vector bias_v_t; + }; + public: AdamOptimizer(Rows rows, Cols cols, double start_learning_rate = kDefaultStartLearningRate, double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); - Matrix ComputeCorrection(const Matrix& gradient); + void GetCorrection(Matrix& weights_gradient, Vector& bias_gradient); private: static constexpr double kDefaultStartLearningRate = 3e-4; @@ -20,7 +27,6 @@ class AdamOptimizer { double learning_rate_; double beta1_; double beta2_; - Matrix m_t_; - Matrix v_t_; + Moments moments_; }; } // namespace network \ No newline at end of file diff --git a/src/dataloader.cpp b/src/dataloader.cpp index 36ff139..ea1e756 100644 --- a/src/dataloader.cpp +++ b/src/dataloader.cpp @@ -3,12 +3,14 @@ #include #include #include +#include +#include namespace network { namespace details { Shuffle::Shuffle() : gen_(kDefaultSeed) { } -Shuffle::Shuffle(int seed) : gen_(kDefaultSeed) { +Shuffle::Shuffle(int seed) : gen_(seed) { } void Shuffle::ShuffleData(Index begin, Index end, Data& data) { assert(begin >= 0 && "Negative start index"); @@ -42,12 +44,17 @@ int DataLoader::Size() const { std::vector DataLoader::Batches(int batch_size) const { assert(batch_size > 0 && "Batch size must be positive"); assert(batch_size <= Size() && "Batch size exceeds training data size"); + auto start = std::chrono::high_resolution_clock::now(); std::vector batches; int data_size = Size(); for (int i = 0; i < data_size; i += batch_size) { int cur_batch_size = std::min(batch_size, data_size - i); batches.push_back(GetBatch(i, cur_batch_size)); } + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + std::cout << "batches time " << duration.count() << std::endl; + return batches; } diff --git a/src/layer.cpp b/src/layer.cpp index 56db892..fee6ed4 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,6 +1,7 @@ #include "layer.h" #include "activation_function.h" #include +#include namespace network { namespace details { @@ -10,17 +11,15 @@ Random::Random(int seed) : generator_(seed) { Matrix Random::NormalMatrix(Index rows, Index cols, double mean, double stdev) { return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); } -Vector Random::NormalVector(Index rows, double mean, double stdev) { - return Eigen::Rand::normal(rows, 1, generator_, mean, stdev); + +Matrix Random::ConstMatrix(Index rows, Index cols, double value) { + return Eigen::MatrixXd::Constant(rows, cols, value); } } // namespace details -Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd) - : weights_( - InitializedWeights(static_cast(output_size), static_cast(input_size), rnd)), - bias_(InitializedBias(static_cast(output_size), rnd)), - func_(name) { +Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd) : func_(name) { + InitializeParametrs(static_cast(output_size), static_cast(input_size), name, rnd); } Layer::Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) : weights_(weights), bias_(bias), func_(name) { @@ -33,12 +32,6 @@ Matrix Layer::ApplyLinear(const Matrix& input_batch) const { return output_batch; } -// Vector Layer::Forward(const Vector& input) { -// assert(input.size() == weights_.cols() && "wrong size of layer"); -// Vector output = weights_ * input + bias_; -// output = func_.Activate(output); -// return output; -// } Matrix Layer::Forward(const Matrix& input) const { return func_.Apply(ApplyLinear(input)); } @@ -91,11 +84,27 @@ Index Layer::GetWeightRows() const { return weights_.rows(); } -Matrix Layer::InitializedWeights(Index rows, Index cols, Rand& rnd) { - return rnd.NormalMatrix(rows, cols); -} -Vector Layer::InitializedBias(Index rows, Rand& rnd) { - return rnd.NormalVector(rows); +void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name, Rand& rnd) { + constexpr double kConst = 0.01; + double stdev; + switch (name) { + case ActivationFunc::Name::ReLU: + stdev = std::sqrt(2.0 / static_cast(cols)); + weights_ = rnd.NormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.ConstMatrix(rows, 1, kConst); + break; + + case ActivationFunc::Name::Linear: + weights_ = rnd.NormalMatrix(rows, cols, 0, kConst); + bias_ = rnd.ConstMatrix(rows, 1, 0); + break; + + default: + stdev = std::sqrt(2.0 / (static_cast(cols) + static_cast(rows))); + weights_ = rnd.NormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.ConstMatrix(rows, 1, 0); + break; + } } Layer::Rand& Layer::GlobalRandom() { static Rand rnd; diff --git a/src/layer.h b/src/layer.h index ab06a9e..ac00467 100644 --- a/src/layer.h +++ b/src/layer.h @@ -1,8 +1,7 @@ #pragma once -#include "activation_function.h" - #include "linalg.h" +#include "activation_function.h" namespace network { @@ -17,9 +16,8 @@ class Random { public: Random(int seed = kDefaultSeed); - // лучше нормальное или равномерное? Matrix NormalMatrix(Index rows, Index cols, double mean = 0, double stdev = 1); - Vector NormalVector(Index rows, double mean = 0, double stdev = 1); + Matrix ConstMatrix(Index rows, Index cols, double value); private: static constexpr int kDefaultSeed = 42; @@ -33,29 +31,27 @@ class Layer { using Rand = details::Random; public: + Layer() = default; Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd = GlobalRandom()); // этот конструктор больше нужен для тестирование Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); Matrix ApplyLinear(const Matrix& input) const; - // Vector Forward(const Vector& input); Matrix Forward(const Matrix& input) const; - // Vector Backward(const Vector& gradient); Matrix Backward(const Matrix& input_batch, const Matrix& gradient) const; - - // наверно это плохая идея, но я думал так, чтобы не пересчитывать матрицу градиентов по b; Эта - // структора сделана лишь с этой целью. WeightsBiasGradient GetWeightsBiasGradient(const Matrix& input_batch, const Matrix& gradient) const; void UpdateWeights(const Matrix& correction); void UpdateBias(const Vector correction); Index GetWeightCols() const; Index GetWeightRows() const; + // friend FileWriter& operator<<(FileWriter& in, const Layer& layer); + // friend FileReader& operator>>(FileReader& out, Layer& layer); private: static Rand& GlobalRandom(); - Matrix InitializedWeights(Index rows, Index cols, Rand& rnd); - Vector InitializedBias(Index rows, Rand& rnd); + void InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name, Rand& rnd); + ActivationFunc func_; Matrix weights_; Vector bias_; diff --git a/src/main.cpp b/src/main.cpp index f36b97b..6ac4017 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,4 +1,5 @@ +#include "activation_function.h" #include "loss_function.h" #include "net.h" #include @@ -6,11 +7,11 @@ #include #include "mnist/mnist_reader.hpp" -//за это сори, я по фасту накидывал, чтобы протестить +// за это сори, я по фасту накидывал, чтобы протестить using namespace network; Matrix ConvertToMatrix(const std::vector>& images) { size_t num_images = images.size(); - Matrix eigen_images(784, num_images); + Matrix eigen_images(784, num_images); for (size_t i = 0; i < num_images; ++i) { for (int j = 0; j < 784; ++j) { @@ -22,11 +23,11 @@ Matrix ConvertToMatrix(const std::vector>& images) { } Matrix LabelToVector(const std::vector& labels) { size_t num_labels = labels.size(); - Matrix onehot(10, num_labels); + Matrix onehot(10, num_labels); onehot.setZero(); for (size_t i = 0; i < num_labels; ++i) { - onehot(labels[i], i) = 1.0; + onehot(labels[i], i) = 1.0; } return onehot; @@ -47,14 +48,17 @@ std::pair GetTrainDataSet(std::string path_to_data) { int main() { std::string path = "../libs/mnist"; std::pair ans = GetTrainDataSet(path); + DataLoader dl(std::move(ans.first)); - Net nn(std::move(dl), LossFunc::Name::CrossEntropy); - nn.AddLayer(In{784}, Out{256}, ActivationFunc::Name::ReLU); - nn.AddLayer(In{256}, Out{10}, ActivationFunc::Name::Softmax); - nn.Backward(64, 15); + std::vector layers{784, 256, 10}; + std::vector names{ActivationFunc::Name::ReLU, + ActivationFunc::Name::Softmax}; + Net nn(layers, names); + nn.Backward(dl, LossFunc::Name::CrossEntropy, 60, 17); + std::cout << "///////////////////////////////// TEST PATR//////////////////////////////////////" << std::endl; - Matrix evalute_test = nn.Evalute(ans.second.input); + Matrix evalute_test = nn.Evaluate(ans.second.input); int size_test = ans.second.input.cols(); int cnt = 0; std::cout << " Size TESt " << size_test << std::endl; @@ -68,5 +72,5 @@ int main() { } } double accuracy = (static_cast(cnt) / size_test) * 100; - std::cout<< accuracy << std::endl; + std::cout << accuracy << std::endl; } diff --git a/src/net.cpp b/src/net.cpp index 79d6462..4db081a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,19 +1,24 @@ #include "net.h" -#include #include "activation_function.h" #include "adam_optimizer.h" #include "layer.h" + #include +#include + namespace network { -Net::Net(DataLoader&& dl, LossFunc::Name name) : dl_(std::move(dl)), loss_func_(name) { -} -void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { - assert((layers_.empty() || layers_.back().GetWeightRows() == static_cast(input_size)) && - "incorrect layer"); - layers_.emplace_back(In{input_size}, Out{output_size}, name); +Net::Net(std::vector sizes_of_layers, + std::vector names_of_activation_functions) { + assert(!sizes_of_layers.empty() && "sizes_of_layers empty"); + assert(sizes_of_layers.size() - names_of_activation_functions.size() == 1 && + "invalid parametrs"); + for (int i = 1; i < sizes_of_layers.size(); ++i) { + AddLayer(In{sizes_of_layers[i - 1]}, Out{sizes_of_layers[i]}, + names_of_activation_functions[i - 1]); + } } -Net::ComputedBatches Net::Forward(const Matrix& batch) const { - Matrix cur_input = batch; + +Net::ComputedBatches Net::Forward(Matrix& cur_input) const { std::vector inputs; inputs.push_back(cur_input); for (const Layer& layer : layers_) { @@ -23,7 +28,7 @@ Net::ComputedBatches Net::Forward(const Matrix& batch) const { return inputs; } -Matrix Net::Evalute(const Matrix& batch) const { +Matrix Net::Evaluate(const Matrix& batch) const { Matrix cur_batch = batch; for (const Layer& layer : layers_) { cur_batch = layer.Forward(cur_batch); @@ -31,41 +36,46 @@ Matrix Net::Evalute(const Matrix& batch) const { return cur_batch; } -void Net::Backward(int batch_size, int num_epochs, double start_learning_rate, double beta1, - double beta2) { +void Net::Backward(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, + double start_learning_rate, double beta1, double beta2) { assert(!layers_.empty() && "empty layers"); + std::vector optimizers; + LossFunc loss_func(name); for (int i = 0; i < layers_.size(); ++i) { - Index rows = layers_[i].GetWeightRows(); Index cols = layers_[i].GetWeightCols(); AdamOptimizer opt_of_weights(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); - AdamOptimizer opt_of_bias(Rows{rows}, Cols{1}, start_learning_rate, beta1, beta2); - opts_.push_back({opt_of_weights, opt_of_bias}); + optimizers.emplace_back(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); } + for (int epoch = 0; epoch < num_epochs; ++epoch) { - dl_.ShuffleData(); - for (const Data& train_batch : dl_.Batches(batch_size)) { - TrainBatch(train_batch); + std::cout << "EPOCH " << epoch << std::endl; + + dl.ShuffleData(); + for (Data& train_batch : dl.Batches(batch_size)) { + TrainBatch(train_batch, optimizers, loss_func); } } } - -void Net::TrainBatch(const Data& data) { +void Net::TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func) { assert(!layers_.empty() && "no layers"); - assert(layers_.size() == opts_.size() && "different sizes of optimizers and layers"); + assert(layers_.size() == optimizers.size() && "different sizes of optimizers and layers"); ComputedBatches computed_batces = Forward(data.input); assert(computed_batces.size() - layers_.size() == 1 && "invalide size of layers_ or computed_batches"); - Matrix cur_gradient = loss_func_.GetGradient(computed_batces.back(), data.output); + Matrix cur_gradient = loss_func.GetGradient(computed_batces.back(), data.output); for (int i = layers_.size() - 1; i >= 0; --i) { WeightsBiasGradient grad = layers_[i].GetWeightsBiasGradient(computed_batces[i], cur_gradient); - Matrix correction_weights = opts_[i].weights.ComputeCorrection(grad.weights); - Vector correction_bias = opts_[i].bias.ComputeCorrection(grad.bias); + optimizers[i].GetCorrection(grad.weights, grad.bias); cur_gradient = layers_[i].Backward(computed_batces[i], cur_gradient); - layers_[i].UpdateWeights(correction_weights); - layers_[i].UpdateBias(correction_bias); + layers_[i].UpdateWeights(grad.weights); + layers_[i].UpdateBias(grad.bias); } } - +void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { + assert((layers_.empty() || layers_.back().GetWeightRows() == static_cast(input_size)) && + "incorrect layer"); + layers_.emplace_back(input_size, output_size, name); +} } // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h index 8b33024..5b66b4a 100644 --- a/src/net.h +++ b/src/net.h @@ -5,6 +5,7 @@ #include "loss_function.h" #include "dataloader.h" #include "adam_optimizer.h" + #include namespace network { @@ -22,22 +23,20 @@ class Net { using Optimizers = std::vector; public: - Net(DataLoader&& dl, LossFunc::Name name); - void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); - ComputedBatches Forward(const Matrix& batch) const; - Matrix Evalute(const Matrix& batch) const; - void Backward(int batch_size, int num_epochs, + Net(std::vector sizes_of_layers, + std::vector names_of_activation_functions); + Matrix Evaluate(const Matrix& batch) const; + void Backward(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, double start_learning_rate = kDefaultStartLearningRate, double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); private: - void TrainBatch(const Data& data); + ComputedBatches Forward(Matrix& batch) const; + void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); + void TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func); static constexpr double kDefaultStartLearningRate = 3e-4; static constexpr double kDefaultBeta1 = 0.9; static constexpr double kDefaultBeta2 = 0.99; - DataLoader dl_; - LossFunc loss_func_; Layers layers_; - Optimizers opts_; }; } // namespace network \ No newline at end of file From 3af6ba81d0fc96cdd09b6042c4d3f73f08738018 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Thu, 10 Apr 2025 16:03:54 +0300 Subject: [PATCH 09/13] 98% on mnist test --- .gitignore | 4 +- CMakeLists.txt | 8 +-- src/activation_function.cpp | 25 +++++---- src/activation_function.h | 7 +-- src/file_reader_writer.h | 105 ++++++++++++++++++++++++++++++++++++ src/layer.cpp | 27 ++++++++-- src/layer.h | 12 +++-- src/main.cpp | 76 -------------------------- src/net.cpp | 102 +++++++++++++++++++++++++++-------- src/net.h | 38 ++++++++++--- tests/mnist_utils.cpp | 68 +++++++++++++++++++++++ tests/mnist_utils.h | 25 +++++++++ tests/save_net.bin | 0 tests/test_mnist.cpp | 46 ++++++++++++++++ 14 files changed, 408 insertions(+), 135 deletions(-) create mode 100644 src/file_reader_writer.h delete mode 100644 src/main.cpp create mode 100644 tests/mnist_utils.cpp create mode 100644 tests/mnist_utils.h create mode 100644 tests/save_net.bin create mode 100644 tests/test_mnist.cpp diff --git a/.gitignore b/.gitignore index 3a5f27a..e05a69a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,5 @@ compile_commands.json .vscode* build* libs/eigen -libs/tqdm.cpp -src/file_reader_writer* + + diff --git a/CMakeLists.txt b/CMakeLists.txt index e6ed887..2c5bb58 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) project(NeuralNetwork LANGUAGES CXX) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -12,7 +12,7 @@ include_directories( ${CMAKE_SOURCE_DIR}/libs/EigenRand ${CMAKE_SOURCE_DIR}/libs/googletest/googletest/include ${CMAKE_SOURCE_DIR}/libs/mnist/include - ${CMAKE_SOURCE_DIR}/src/include + ${CMAKE_SOURCE_DIR}/src ) @@ -43,13 +43,15 @@ target_link_libraries(test_layer PRIVATE gtest gtest_main) add_executable(main src/main.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp) - +add_executable(test_mnist tests/test_mnist.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp tests/mnist_utils.cpp) +target_link_libraries(test_mnist PRIVATE gtest gtest_main) enable_testing() add_test(NAME test_activation_function COMMAND test_activation_function) add_test(NAME test_loss_function COMMAND test_loss_function) add_test(NAME test_layer COMMAND test_layer) +add_test(NAME test_mnist COMMAND test_mnist) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index 7c9280a..fc8cf59 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -46,7 +46,7 @@ struct Softmax { return tmp - applied_softmax * applied_softmax.transpose(); } }; -struct Linear { +struct Id { static Vector Apply(const Vector& x) { return x; } @@ -60,29 +60,29 @@ struct Linear { void ActivationFunc::SetFunction(ActivationFunc::Name name) { switch (name) { case ActivationFunc::Name::Sigmoid: - id_ = static_cast(ActivationFunc::Name::Sigmoid); + func_id_ = static_cast(ActivationFunc::Name::Sigmoid); apply_ = details::Sigmoid::Apply; differential_ = details::Sigmoid::GetDifferential; break; case ActivationFunc::Name::ReLU: - id_ = static_cast(ActivationFunc::Name::ReLU); + func_id_ = static_cast(ActivationFunc::Name::ReLU); apply_ = details::ReLU::Apply; differential_ = details::ReLU::GetDifferential; break; case ActivationFunc::Name::Tanh: - id_ = static_cast(ActivationFunc::Name::Tanh); + func_id_ = static_cast(ActivationFunc::Name::Tanh); apply_ = details::Tanh::Apply; differential_ = details::Tanh::GetDifferential; break; case ActivationFunc::Name::Softmax: - id_ = static_cast(ActivationFunc::Name::Softmax); + func_id_ = static_cast(ActivationFunc::Name::Softmax); apply_ = details::Softmax::Apply; differential_ = details::Softmax::GetDifferential; break; - case ActivationFunc::Name::Linear: - id_ = static_cast(ActivationFunc::Name::Linear); - apply_ = details::Linear::Apply; - differential_ = details::Linear::GetDifferential; + case ActivationFunc::Name::Id: + func_id_ = static_cast(ActivationFunc::Name::Id); + apply_ = details::Id::Apply; + differential_ = details::Id::GetDifferential; break; default: @@ -106,7 +106,10 @@ Matrix ActivationFunc::Apply(const Matrix& x) const { Matrix ActivationFunc::GetDifferential(const Vector& x) const { return differential_(x); } -int ActivationFunc::GetId() const { - return id_; +int ActivationFunc::GetFuncId() const { + return func_id_; +} +bool ActivationFunc::operator==(const ActivationFunc& other) const { + return GetFuncId() == other.GetFuncId(); } } // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h index 1a398ac..5eba2d4 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -10,17 +10,18 @@ class ActivationFunc { public: ActivationFunc() = default; - enum class Name { Sigmoid, ReLU, Tanh, Softmax, Linear }; + enum class Name { Sigmoid, ReLU, Tanh, Softmax, Id }; explicit ActivationFunc(Name name); void SetFunction(Name name); Matrix Apply(const Matrix& x) const; Matrix GetDifferential(const Vector& x) const; - int GetId() const; + int GetFuncId() const; + bool operator==(const ActivationFunc& other) const; private: Function apply_; Differential differential_; - int id_; + int func_id_; }; } // namespace network diff --git a/src/file_reader_writer.h b/src/file_reader_writer.h new file mode 100644 index 0000000..3ede44f --- /dev/null +++ b/src/file_reader_writer.h @@ -0,0 +1,105 @@ + +#pragma once +#include +#include + +#include "linalg.h" + +namespace network { + +class FileWriter { +public: + explicit FileWriter(const std::string& filename) : file_(filename, std::ios::binary) { + } + + template + typename std::enable_if_t, FileWriter&> operator<<(const T& value) { + file_.write(reinterpret_cast(&value), sizeof(T)); + return *this; + } + + template + FileWriter& operator<<(const std::vector& vec) { + size_t size = vec.size(); + *this << size; + for (const auto& element : vec) { + *this << element; + } + return *this; + } + + FileWriter& operator<<(const Matrix& mat) { + Matrix::Index rows = mat.rows(); + Matrix::Index cols = mat.cols(); + file_.write(reinterpret_cast(&rows), sizeof(Index)); + file_.write(reinterpret_cast(&cols), sizeof(Index)); + file_.write(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + return *this; + } + + FileWriter& operator<<(const Vector& vec) { + Vector::Index size = vec.size(); + file_.write(reinterpret_cast(&size), sizeof(Index)); + file_.write(reinterpret_cast(vec.data()), size * sizeof(double)); + return *this; + } + void CloseFile() { + file_.close(); + } + +private: + std::ofstream file_; +}; + +class FileReader { +public: + explicit FileReader(const std::string& filename) : file_(filename, std::ios::binary) { + } + + template + typename std::enable_if_t, FileReader&> operator>>(T& value) { + file_.read(reinterpret_cast(&value), sizeof(T)); + return *this; + } + + template + FileReader& operator>>(std::vector& vec) { + vec.clear(); + size_t size; + *this >> size; + vec.reserve(size); + T element; + for (size_t i = 0; i < size; ++i) { + + *this >> element; + vec.push_back(std::move(element)); + } + + return *this; + } + + FileReader& operator>>(Matrix& mat) { + Index rows, cols; + file_.read(reinterpret_cast(&rows), sizeof(Index)); + file_.read(reinterpret_cast(&cols), sizeof(Index)); + mat.resize(rows, cols); + file_.read(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + return *this; + } + + FileReader& operator>>(Vector& vec) { + Index size; + file_.read(reinterpret_cast(&size), sizeof(Index)); + vec.resize(size); + file_.read(reinterpret_cast(vec.data()), size * sizeof(double)); + return *this; + } + void CloseFile() { + file_.close(); + } + +private: + std::ifstream file_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/layer.cpp b/src/layer.cpp index fee6ed4..f88e20b 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,5 +1,6 @@ #include "layer.h" #include "activation_function.h" +#include "file_reader_writer.h" #include #include namespace network { @@ -77,12 +78,30 @@ WeightsBiasGradient Layer::GetWeightsBiasGradient(const Matrix& input_batch, grad.weights = matrix_grad_biases * input_batch.transpose() / matrix_grad_biases.cols(); return grad; } -Index Layer::GetWeightCols() const { - return weights_.cols(); + +FileWriter& operator<<(FileWriter& in, const Layer& layer) { + in << layer.func_.GetFuncId(); + in << layer.weights_; + in << layer.bias_; + return in; +} + +FileReader& operator>>(FileReader& out, Layer& layer) { + int id; + Matrix weights; + Vector bias; + out >> id; + out >> weights; + out >> bias; + layer = Layer(weights, bias, static_cast(id)); + return out; } -Index Layer::GetWeightRows() const { +Index Layer::GetWeightsRows() const { return weights_.rows(); } +Index Layer::GetWeightsCols() const { + return weights_.cols(); +} void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name, Rand& rnd) { constexpr double kConst = 0.01; @@ -94,7 +113,7 @@ void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name nam bias_ = rnd.ConstMatrix(rows, 1, kConst); break; - case ActivationFunc::Name::Linear: + case ActivationFunc::Name::Id: weights_ = rnd.NormalMatrix(rows, cols, 0, kConst); bias_ = rnd.ConstMatrix(rows, 1, 0); break; diff --git a/src/layer.h b/src/layer.h index ac00467..e1a4e4e 100644 --- a/src/layer.h +++ b/src/layer.h @@ -2,7 +2,7 @@ #pragma once #include "linalg.h" #include "activation_function.h" - +#include "file_reader_writer.h" namespace network { struct WeightsBiasGradient { @@ -43,10 +43,12 @@ class Layer { const Matrix& gradient) const; void UpdateWeights(const Matrix& correction); void UpdateBias(const Vector correction); - Index GetWeightCols() const; - Index GetWeightRows() const; - // friend FileWriter& operator<<(FileWriter& in, const Layer& layer); - // friend FileReader& operator>>(FileReader& out, Layer& layer); + + friend FileWriter& operator<<(FileWriter& in, const Layer& layer); + friend FileReader& operator>>(FileReader& out, Layer& layer); + bool operator==(const Layer& other) const = default; + Index GetWeightsRows() const; + Index GetWeightsCols() const; private: static Rand& GlobalRandom(); diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index 6ac4017..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,76 +0,0 @@ - -#include "activation_function.h" -#include "loss_function.h" -#include "net.h" -#include -#include -#include -#include "mnist/mnist_reader.hpp" - -// за это сори, я по фасту накидывал, чтобы протестить -using namespace network; -Matrix ConvertToMatrix(const std::vector>& images) { - size_t num_images = images.size(); - Matrix eigen_images(784, num_images); - - for (size_t i = 0; i < num_images; ++i) { - for (int j = 0; j < 784; ++j) { - eigen_images(j, i) = static_cast(images[i][j]) / 255.0; - } - } - - return eigen_images; -} -Matrix LabelToVector(const std::vector& labels) { - size_t num_labels = labels.size(); - Matrix onehot(10, num_labels); - onehot.setZero(); - - for (size_t i = 0; i < num_labels; ++i) { - onehot(labels[i], i) = 1.0; - } - - return onehot; -} - -std::pair GetTrainDataSet(std::string path_to_data) { - auto dataset = mnist::read_dataset(path_to_data); - assert(!dataset.training_images.empty() && "empty dataset"); - Matrix train_images = ConvertToMatrix(dataset.training_images); - Matrix train_labels = LabelToVector(dataset.training_labels); - Data train{train_images, train_labels}; - Matrix test_images = ConvertToMatrix(dataset.test_images); - Matrix test_labels = LabelToVector(dataset.test_labels); - Data test_data{test_images, test_labels}; - return std::make_pair(train, test_data); -} - -int main() { - std::string path = "../libs/mnist"; - std::pair ans = GetTrainDataSet(path); - - DataLoader dl(std::move(ans.first)); - std::vector layers{784, 256, 10}; - std::vector names{ActivationFunc::Name::ReLU, - ActivationFunc::Name::Softmax}; - Net nn(layers, names); - nn.Backward(dl, LossFunc::Name::CrossEntropy, 60, 17); - - std::cout << "///////////////////////////////// TEST PATR//////////////////////////////////////" - << std::endl; - Matrix evalute_test = nn.Evaluate(ans.second.input); - int size_test = ans.second.input.cols(); - int cnt = 0; - std::cout << " Size TESt " << size_test << std::endl; - Index max_our_ind; - Index max_test_ind; - for (int i = 0; i < size_test; ++i) { - evalute_test.col(i).maxCoeff(&max_our_ind); - ans.second.output.col(i).maxCoeff(&max_test_ind); - if (max_test_ind == max_our_ind) { - ++cnt; - } - } - double accuracy = (static_cast(cnt) / size_test) * 100; - std::cout << accuracy << std::endl; -} diff --git a/src/net.cpp b/src/net.cpp index 4db081a..dc7f7b0 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,20 +1,29 @@ + +#include +#include +#include #include "net.h" #include "activation_function.h" #include "adam_optimizer.h" #include "layer.h" -#include -#include - namespace network { -Net::Net(std::vector sizes_of_layers, - std::vector names_of_activation_functions) { - assert(!sizes_of_layers.empty() && "sizes_of_layers empty"); - assert(sizes_of_layers.size() - names_of_activation_functions.size() == 1 && - "invalid parametrs"); - for (int i = 1; i < sizes_of_layers.size(); ++i) { - AddLayer(In{sizes_of_layers[i - 1]}, Out{sizes_of_layers[i]}, - names_of_activation_functions[i - 1]); +Net::Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params) { + assert(!layer_sizes.empty() && "layer_sizes empty"); + assert(layer_sizes.size() - activation_functions.size() == 1 && "invalid parametrs"); + assert(layer_params.empty() || activation_functions.size() == layer_params.size() && + "invalid sizes of layer_params or activation_functions"); + if (layer_params.empty()) { + for (int i = 1; i < layer_sizes.size(); ++i) { + AddLayer(In{layer_sizes[i - 1]}, Out{layer_sizes[i]}, activation_functions[i - 1]); + } + + } else { + CheckParams(layer_sizes, activation_functions, layer_params); + for (int i = 0; i < layer_params.size(); ++i) { + AddLayer(layer_params[i].weights, layer_params[i].bias, activation_functions[i]); + } } } @@ -36,27 +45,79 @@ Matrix Net::Evaluate(const Matrix& batch) const { return cur_batch; } -void Net::Backward(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, - double start_learning_rate, double beta1, double beta2) { +void Net::Train(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, Info info, + double start_learning_rate, double beta1, double beta2) { assert(!layers_.empty() && "empty layers"); + auto start = std::chrono::high_resolution_clock::now(); std::vector optimizers; LossFunc loss_func(name); for (int i = 0; i < layers_.size(); ++i) { - Index rows = layers_[i].GetWeightRows(); - Index cols = layers_[i].GetWeightCols(); - AdamOptimizer opt_of_weights(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); + Index rows = layers_[i].GetWeightsRows(); + Index cols = layers_[i].GetWeightsCols(); optimizers.emplace_back(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); } + if (info == Info::On) { - for (int epoch = 0; epoch < num_epochs; ++epoch) { - std::cout << "EPOCH " << epoch << std::endl; + for (int epoch = 0; epoch < num_epochs; ++epoch) { + auto start = std::chrono::high_resolution_clock::now(); + + std::cout << "Epoch" << " " << epoch + 1 << std::endl; + + dl.ShuffleData(); + for (Data& train_batch : dl.Batches(batch_size)) { + TrainBatch(train_batch, optimizers, loss_func); + } + auto stop = std::chrono::high_resolution_clock::now(); + auto time = stop - start; + auto hours = std::chrono::duration_cast(time); + auto minutes = + std::chrono::duration_cast(time % std::chrono::hours(1)); + auto seconds = + std::chrono::duration_cast(time % std::chrono::minutes(1)); + std::cout << "Epoch time:" << " " << std::setfill('0') << std::setw(2) << hours.count() + << ":" << std::setw(2) << minutes.count() << ":" << std::setw(2) + << seconds.count() << "\n"; + } + return; + } + for (int epoch = 0; epoch < num_epochs; ++epoch) { dl.ShuffleData(); for (Data& train_batch : dl.Batches(batch_size)) { TrainBatch(train_batch, optimizers, loss_func); } } } +FileWriter& operator<<(FileWriter& out, const Net& net) { + out << net.layers_; + return out; +} +FileReader& operator>>(FileReader& in, Net& net) { + + in >> net.layers_; + return in; +} + +void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { + assert((layers_.empty() || layers_.back().GetWeightsRows() == static_cast(input_size)) && + "incorrect layer"); + layers_.emplace_back(input_size, output_size, name); +} +void Net::AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) { + assert(weights.cols() == bias.rows() && "bad params"); + layers_.emplace_back(weights, bias, name); +} + +void Net::CheckParams(const LayerSizes& layer_sizes, + const ActivationFunctions& activation_functions, const Params& layer_params) { + assert(layer_params.size() == activation_functions.size() && "wrong paramers"); + for (int i = 0; i < layer_sizes.size() - 1; ++i) { + assert(layer_sizes[i] == layer_params[i].weights.cols() && "wrong params"); + assert(layer_sizes[i + 1] == layer_params[i].weights.rows() && "wrong params"); + assert(layer_sizes[i + 1] == layer_params[i].bias.size()); + } +} + void Net::TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func) { assert(!layers_.empty() && "no layers"); assert(layers_.size() == optimizers.size() && "different sizes of optimizers and layers"); @@ -73,9 +134,4 @@ void Net::TrainBatch(Data& data, std::vector& optimizers, LossFun layers_[i].UpdateBias(grad.bias); } } -void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { - assert((layers_.empty() || layers_.back().GetWeightRows() == static_cast(input_size)) && - "incorrect layer"); - layers_.emplace_back(input_size, output_size, name); -} } // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h index 5b66b4a..b5c70b5 100644 --- a/src/net.h +++ b/src/net.h @@ -1,4 +1,6 @@ #pragma once +#include +#include "file_reader_writer.h" #include "linalg.h" #include "activation_function.h" #include "layer.h" @@ -6,8 +8,6 @@ #include "dataloader.h" #include "adam_optimizer.h" -#include - namespace network { namespace details { @@ -16,27 +16,49 @@ struct OptimizersParams { AdamOptimizer bias; }; } // namespace details +struct LayerParams { + Matrix weights; + Vector bias; +}; +enum class Info { On, Off }; class Net { + // using Layers = std::vector; + // using ComputedBatches = std::vector; + // using Optimizers = std::vector; + // using LayerSizes = std::vector; + // using ActivationFunctions = std::vector; + +public: using Layers = std::vector; using ComputedBatches = std::vector; using Optimizers = std::vector; + using LayerSizes = std::vector; + using ActivationFunctions = std::vector; + using Params = std::vector; + Net() = default; + Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params = {}); -public: - Net(std::vector sizes_of_layers, - std::vector names_of_activation_functions); Matrix Evaluate(const Matrix& batch) const; - void Backward(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, - double start_learning_rate = kDefaultStartLearningRate, - double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); + void Train(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, Info info, + double start_learning_rate = kDefaultStartLearningRate, double beta1 = kDefaultBeta1, + double beta2 = kDefaultBeta2); + friend FileWriter& operator<<(FileWriter& out, const Net& net); + friend FileReader& operator>>(FileReader& in, Net& net); + bool operator==(const Net& other) const = default; private: ComputedBatches Forward(Matrix& batch) const; void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); + void AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); + void CheckParams(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params); void TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func); static constexpr double kDefaultStartLearningRate = 3e-4; static constexpr double kDefaultBeta1 = 0.9; static constexpr double kDefaultBeta2 = 0.99; + Layers layers_; }; } // namespace network \ No newline at end of file diff --git a/tests/mnist_utils.cpp b/tests/mnist_utils.cpp new file mode 100644 index 0000000..6ca0492 --- /dev/null +++ b/tests/mnist_utils.cpp @@ -0,0 +1,68 @@ +#include "net.h" +#include +#include "mnist_utils.h" + +namespace network { +MnistUtils::MnistUtils(const std::string& path) : path_(path) { +} + +Data MnistUtils::GetTrainData() { + auto dataset = mnist::read_dataset(path_); + assert(!dataset.training_images.empty() && "empty dataset"); + Matrix train_images = ConvertToMatrix(dataset.training_images); + Matrix train_labels = LabelToMatrix(dataset.training_labels); + return {train_images, train_labels}; +} +Data MnistUtils::GetTestData() { + auto dataset = mnist::read_dataset(path_); + Matrix test_images = ConvertToMatrix(dataset.test_images); + Matrix test_labels = LabelToMatrix(dataset.test_labels); + return {test_images, test_labels}; +} + +Matrix MnistUtils::ConvertToMatrix(const std::vector>& images) { + size_t num_images = images.size(); + Matrix eigen_images(784, num_images); + + for (size_t i = 0; i < num_images; ++i) { + for (int j = 0; j < 784; ++j) { + eigen_images(j, i) = static_cast(images[i][j]) / 255.0; + } + } + + return eigen_images; +} + +Matrix MnistUtils::LabelToMatrix(const std::vector& labels) { + size_t num_labels = labels.size(); + Matrix onehot(10, num_labels); + onehot.setZero(); + + for (size_t i = 0; i < num_labels; ++i) { + onehot(labels[i], i) = 1.0; + } + + return onehot; +} +double MnistUtils::ComputeAccuracy(const Net& net) { + Data test_data = GetTestData(); + Matrix net_ans = net.Evaluate(test_data.input); + assert(net_ans.cols() == test_data.output.cols() && "wrong cols"); + assert(net_ans.rows() == test_data.output.rows() && "wrong rows"); + int test_size = test_data.input.cols(); + assert(test_size != 0 && "bad test data"); + int cnt = 0; + Index cur_true_ans; + Index cur_ans; + for (int i = 0; i < test_size; ++i) { + net_ans.col(i).maxCoeff(&cur_ans); + test_data.output.col(i).maxCoeff(&cur_true_ans); + if (cur_ans == cur_true_ans) { + ++cnt; + } + } + double accuracy = static_cast(cnt) / static_cast(test_size); + std::cout << "Accuracy:" << " " << accuracy << std::endl; + return accuracy; +} +} // namespace network diff --git a/tests/mnist_utils.h b/tests/mnist_utils.h new file mode 100644 index 0000000..0ea7d13 --- /dev/null +++ b/tests/mnist_utils.h @@ -0,0 +1,25 @@ +#include "net.h" +#include "mnist/mnist_reader.hpp" + +namespace network { +struct TrainTestData{ + Data train; + Data test; +}; +class MnistUtils { + using Images = std::vector>; + using Labels = std::vector; +public: +MnistUtils(const std::string& path); + Data GetTrainData(); + Data GetTestData(); + double ComputeAccuracy(const Net& net); + +private: + Matrix ConvertToMatrix(const Images& images); + Matrix LabelToMatrix(const Labels& labels); + std::string path_; +}; + + +} // namespace network \ No newline at end of file diff --git a/tests/save_net.bin b/tests/save_net.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_mnist.cpp b/tests/test_mnist.cpp new file mode 100644 index 0000000..7c14249 --- /dev/null +++ b/tests/test_mnist.cpp @@ -0,0 +1,46 @@ +#include +#include +#include "activation_function.h" +#include "dataloader.h" +#include "file_reader_writer.h" +#include "loss_function.h" +#include "mnist_utils.h" +void ClearFile(std::string filename) { + std::ofstream file(filename, std::ios::binary | std::ios::trunc); + file.close(); +} + +TEST(Mnist, WriteRead) { + using namespace network; + std::vector layer_sizes = {784, 256, 10}; + std::vector activation_functions = {ActivationFunc::Name::ReLU, + ActivationFunc::Name::Softmax}; + Net nn(layer_sizes, activation_functions); + std::string path = "../libs/mnist"; + MnistUtils mnist_utils(path); + Data train_data = mnist_utils.GetTrainData(); + DataLoader dl(std::move(train_data)); + constexpr int kNumEpocs = 17; + constexpr int kBatchSize = 64; + nn.Train(dl, LossFunc::Name::CrossEntropy, kBatchSize, kNumEpocs, Info::On); + constexpr double kExpectedAccuracy = 0; + double accuracy = mnist_utils.ComputeAccuracy(nn); + EXPECT_GE(accuracy, kExpectedAccuracy); + + std::string filename = "save_net.bin"; + + ClearFile(filename); + FileWriter w(filename); + w << nn; + w.CloseFile(); + FileReader r(filename); + Net net; + r >> net; + r.CloseFile(); + EXPECT_EQ(nn, net); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file From 7cb1fd02840f140b5507fc809fc51e6448f0e8bd Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Tue, 15 Apr 2025 20:59:40 +0300 Subject: [PATCH 10/13] updated code --- CMakeLists.txt | 2 +- src/activation_function.cpp | 8 ++--- src/activation_function.h | 2 +- src/layer.cpp | 61 ++++++++++++++++++++++--------------- src/layer.h | 30 +++++++++--------- src/loss_function.cpp | 10 ++++++ src/loss_function.h | 2 +- src/net.cpp | 3 +- 8 files changed, 68 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c5bb58..d6bd644 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,7 @@ add_executable(test_layer src/activation_function.cpp) target_link_libraries(test_layer PRIVATE gtest gtest_main) -add_executable(main src/main.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp) + add_executable(test_mnist tests/test_mnist.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp tests/mnist_utils.cpp) target_link_libraries(test_mnist PRIVATE gtest gtest_main) diff --git a/src/activation_function.cpp b/src/activation_function.cpp index fc8cf59..ddcae62 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -1,6 +1,7 @@ #include "activation_function.h" #include +#include "file_reader_writer.h" namespace network { namespace details { struct Sigmoid { @@ -57,7 +58,7 @@ struct Id { } // namespace details -void ActivationFunc::SetFunction(ActivationFunc::Name name) { +ActivationFunc::ActivationFunc(ActivationFunc::Name name) { switch (name) { case ActivationFunc::Name::Sigmoid: func_id_ = static_cast(ActivationFunc::Name::Sigmoid); @@ -90,10 +91,6 @@ void ActivationFunc::SetFunction(ActivationFunc::Name name) { } } -ActivationFunc::ActivationFunc(ActivationFunc::Name name) { - SetFunction(name); -} - Matrix ActivationFunc::Apply(const Matrix& x) const { Matrix res(x.rows(), x.cols()); @@ -112,4 +109,5 @@ int ActivationFunc::GetFuncId() const { bool ActivationFunc::operator==(const ActivationFunc& other) const { return GetFuncId() == other.GetFuncId(); } + } // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h index 5eba2d4..627c793 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -2,6 +2,7 @@ #include #include #include "linalg.h" +#include "file_reader_writer.h" namespace network { class ActivationFunc { @@ -12,7 +13,6 @@ class ActivationFunc { ActivationFunc() = default; enum class Name { Sigmoid, ReLU, Tanh, Softmax, Id }; explicit ActivationFunc(Name name); - void SetFunction(Name name); Matrix Apply(const Matrix& x) const; Matrix GetDifferential(const Vector& x) const; int GetFuncId() const; diff --git a/src/layer.cpp b/src/layer.cpp index f88e20b..11a064e 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,26 +1,42 @@ #include "layer.h" +#include "EigenRand/Dists/Basic.h" #include "activation_function.h" #include "file_reader_writer.h" #include #include namespace network { -namespace details { +namespace { +RandomParams& GetRandomGenerator() { + static RandomParams rnd; + return rnd; +} +} // namespace -Random::Random(int seed) : generator_(seed) { +RandomParams::RandomParams(int seed) : generator_(seed) { } -// кажется ты говорил что лучше заполнять нормальным распределением, а не равномерным -Matrix Random::NormalMatrix(Index rows, Index cols, double mean, double stdev) { + +Matrix RandomParams::GenerateNormalMatrix(Index rows, Index cols, double mean, double stdev) { return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); } +Vector RandomParams::GenerateNormalVector(Index rows, double mean, double stdev) { + return GenerateNormalMatrix(rows, 1, mean, stdev); +} +Matrix RandomParams::GenerateUniformMatrix(Index rows, Index cols, double min, double max) { + return Eigen::Rand::uniformReal(rows, cols, generator_, min, max); +} +Vector RandomParams::GenerateUniformVector(Index rows, double min, double max) { + return GenerateUniformMatrix(rows, 1, min, max); +} -Matrix Random::ConstMatrix(Index rows, Index cols, double value) { +Matrix RandomParams::GenerateConstantMatrix(Index rows, Index cols, double value) { return Eigen::MatrixXd::Constant(rows, cols, value); } +Vector RandomParams::GenerateConstantVector(Index rows, double value) { + return GenerateConstantMatrix(rows, 1, value); +} -} // namespace details - -Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd) : func_(name) { - InitializeParametrs(static_cast(output_size), static_cast(input_size), name, rnd); +Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name) : func_(name) { + InitializeParametrs(static_cast(output_size), static_cast(input_size), name); } Layer::Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) : weights_(weights), bias_(bias), func_(name) { @@ -42,7 +58,7 @@ void Layer::UpdateWeights(const Matrix& correction) { "invalid correction"); weights_ -= correction; } -void Layer::UpdateBias(const Vector correction) { +void Layer::UpdateBias(const Vector& correction) { assert(correction.rows() == bias_.rows() && "invalid correction"); bias_ -= correction; } @@ -62,12 +78,11 @@ Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const return new_gradient; } -WeightsBiasGradient Layer::GetWeightsBiasGradient(const Matrix& input_batch, - const Matrix& gradient) const { +ParamsGrad Layer::GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const { assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); - WeightsBiasGradient grad; + ParamsGrad grad; Matrix applied_linear = ApplyLinear(input_batch); Matrix matrix_grad_biases(bias_.rows(), input_batch.cols()); for (int i = 0; i < input_batch.cols(); ++i) { @@ -103,30 +118,28 @@ Index Layer::GetWeightsCols() const { return weights_.cols(); } -void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name, Rand& rnd) { +void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name) { + RandomParams rnd = GetRandomGenerator(); constexpr double kConst = 0.01; double stdev; switch (name) { case ActivationFunc::Name::ReLU: stdev = std::sqrt(2.0 / static_cast(cols)); - weights_ = rnd.NormalMatrix(rows, cols, 0, stdev); - bias_ = rnd.ConstMatrix(rows, 1, kConst); + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.GenerateConstantVector(rows, kConst); break; case ActivationFunc::Name::Id: - weights_ = rnd.NormalMatrix(rows, cols, 0, kConst); - bias_ = rnd.ConstMatrix(rows, 1, 0); + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, kConst); + bias_ = rnd.GenerateConstantVector(rows, 0); break; default: stdev = std::sqrt(2.0 / (static_cast(cols) + static_cast(rows))); - weights_ = rnd.NormalMatrix(rows, cols, 0, stdev); - bias_ = rnd.ConstMatrix(rows, 1, 0); + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.GenerateConstantVector(rows, 0); break; } } -Layer::Rand& Layer::GlobalRandom() { - static Rand rnd; - return rnd; -} + } // namespace network \ No newline at end of file diff --git a/src/layer.h b/src/layer.h index e1a4e4e..b95c7da 100644 --- a/src/layer.h +++ b/src/layer.h @@ -5,44 +5,44 @@ #include "file_reader_writer.h" namespace network { -struct WeightsBiasGradient { +struct ParamsGrad { Matrix weights; Vector bias; }; -namespace details { -class Random { +class RandomParams { using Generator = Eigen::Rand::Vmt19937_64; public: - Random(int seed = kDefaultSeed); - Matrix NormalMatrix(Index rows, Index cols, double mean = 0, double stdev = 1); - Matrix ConstMatrix(Index rows, Index cols, double value); + RandomParams(int seed = kDefaultSeed); + Matrix GenerateNormalMatrix(Index rows, Index cols, double mean = 0, double stdev = 1); + Vector GenerateNormalVector(Index rows, double mean = 0.0, double stdev = 1.0); + Matrix GenerateUniformMatrix(Index rows, Index cols, double low = 0.0, double high = 1.0); + Vector GenerateUniformVector(Index rows, double low = 0.0, double high = 0.0); + Matrix GenerateConstantMatrix(Index rows, Index cols, double value); + Vector GenerateConstantVector(Index rows, double value); private: static constexpr int kDefaultSeed = 42; Generator generator_{kDefaultSeed}; }; -} // namespace details enum class In : Index; enum class Out : Index; class Layer { - using Rand = details::Random; public: Layer() = default; - Layer(In input_size, Out output_size, ActivationFunc::Name name, Rand& rnd = GlobalRandom()); - // этот конструктор больше нужен для тестирование + Layer(In input_size, Out output_size, ActivationFunc::Name name); + Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); Matrix ApplyLinear(const Matrix& input) const; Matrix Forward(const Matrix& input) const; Matrix Backward(const Matrix& input_batch, const Matrix& gradient) const; - WeightsBiasGradient GetWeightsBiasGradient(const Matrix& input_batch, - const Matrix& gradient) const; + ParamsGrad GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const; void UpdateWeights(const Matrix& correction); - void UpdateBias(const Vector correction); + void UpdateBias(const Vector& correction); friend FileWriter& operator<<(FileWriter& in, const Layer& layer); friend FileReader& operator>>(FileReader& out, Layer& layer); @@ -51,9 +51,7 @@ class Layer { Index GetWeightsCols() const; private: - static Rand& GlobalRandom(); - void InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name, Rand& rnd); - + void InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name); ActivationFunc func_; Matrix weights_; Vector bias_; diff --git a/src/loss_function.cpp b/src/loss_function.cpp index de90785..c853965 100644 --- a/src/loss_function.cpp +++ b/src/loss_function.cpp @@ -4,6 +4,16 @@ namespace network { namespace details { struct Mse { + struct Mae { + static double GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).cwiseAbs().sum(); + } + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return (y_out - y_expected).unaryExpr([](double x) { return x > 0 ? 1.0 : -1.0; }); + } + }; static double GetValue(const Vector &y_out, const Vector &y_expected) { assert(y_out.size() == y_expected.size() && "Mse GetValue"); diff --git a/src/loss_function.h b/src/loss_function.h index f795167..9042e82 100644 --- a/src/loss_function.h +++ b/src/loss_function.h @@ -5,7 +5,7 @@ namespace network { class LossFunc { public: - enum class Name { Mse, CrossEntropy }; + enum class Name { Mae, Mse, CrossEntropy }; LossFunc(Name name); double Dist(const Vector &y_out, const Vector &y_expected); Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); diff --git a/src/net.cpp b/src/net.cpp index dc7f7b0..59799bc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -126,8 +126,7 @@ void Net::TrainBatch(Data& data, std::vector& optimizers, LossFun "invalide size of layers_ or computed_batches"); Matrix cur_gradient = loss_func.GetGradient(computed_batces.back(), data.output); for (int i = layers_.size() - 1; i >= 0; --i) { - WeightsBiasGradient grad = - layers_[i].GetWeightsBiasGradient(computed_batces[i], cur_gradient); + ParamsGrad grad = layers_[i].GetParametrsGradient(computed_batces[i], cur_gradient); optimizers[i].GetCorrection(grad.weights, grad.bias); cur_gradient = layers_[i].Backward(computed_batces[i], cur_gradient); layers_[i].UpdateWeights(grad.weights); From 0915ac1ab9a867a9bbc73b1f1869101f09a6d75e Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Thu, 17 Apr 2025 20:04:38 +0300 Subject: [PATCH 11/13] eigen 3.4.0 --- .gitmodules | 3 +++ libs/Eigen | 1 + 2 files changed, 4 insertions(+) create mode 160000 libs/Eigen diff --git a/.gitmodules b/.gitmodules index c7531cd..a981fa8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "libs/mnist"] path = libs/mnist url = https://github.com/wichtounet/mnist.git +[submodule "libs/Eigen"] + path = libs/Eigen + url = https://gitlab.com/libeigen/eigen.git diff --git a/libs/Eigen b/libs/Eigen new file mode 160000 index 0000000..3147391 --- /dev/null +++ b/libs/Eigen @@ -0,0 +1 @@ +Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c From 09ed0cc9a542520eac47d6b25ce501a184de52b6 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Fri, 18 Apr 2025 16:10:49 +0300 Subject: [PATCH 12/13] added readme --- .clang-format | 76 ++++++++++++++++++++++- .clang-tidy | 4 +- .gitignore | 2 +- CMakeLists.txt | 35 +---------- README.md | 98 +++++++++++++++++++++++++++++- src/activation_function.cpp | 43 +++++++------ src/activation_function.h | 7 +-- src/adam_optimizer.cpp | 36 +++++------ src/adam_optimizer.h | 30 ++++----- src/dataloader.cpp | 74 ++++++++++------------ src/dataloader.h | 33 +++------- src/file_reader_writer.h | 54 ++++++++-------- src/{linalg.h => global_usings.h} | 12 ++++ src/layer.cpp | 48 +++++---------- src/layer.h | 30 ++------- src/loss_function.cpp | 47 ++++++++------ src/loss_function.h | 16 +++-- src/net.cpp | 49 ++++++++------- src/net.h | 39 ++++-------- src/random_params.cpp | 31 ++++++++++ src/random_params.h | 22 +++++++ tests/mnist_utils.cpp | 24 ++++---- tests/mnist_utils.h | 13 ++-- tests/test_activation_function.cpp | 22 ------- tests/test_dataloader.cpp | 18 ------ tests/test_layer.cpp | 57 ----------------- tests/test_loss_function.cpp | 2 - tests/test_mnist.cpp | 9 +-- 28 files changed, 495 insertions(+), 436 deletions(-) rename src/{linalg.h => global_usings.h} (54%) create mode 100644 src/random_params.cpp create mode 100644 src/random_params.h delete mode 100644 tests/test_activation_function.cpp delete mode 100644 tests/test_dataloader.cpp delete mode 100644 tests/test_layer.cpp delete mode 100644 tests/test_loss_function.cpp diff --git a/.clang-format b/.clang-format index fe699e0..9fcd2d3 100644 --- a/.clang-format +++ b/.clang-format @@ -1,3 +1,4 @@ +Language: Cpp BasedOnStyle: Google IndentWidth: 4 AccessModifierOffset: -4 @@ -6,5 +7,76 @@ AllowShortFunctionsOnASingleLine: None AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false DerivePointerAlignment: true -KeepEmptyLinesAtTheStartOfBlocks: true -SortIncludes: false +KeepEmptyLinesAtTheStartOfBlocks: false +SortIncludes: Never +MaxEmptyLinesToKeep: 1 +InsertTrailingCommas: None +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: None +AlignOperands: Align +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: false +AllowShortLambdasOnASingleLine: Inline +AlwaysBreakAfterReturnType: None +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BreakBeforeBraces: Attach +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterStruct: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +CompactNamespaces: false +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +EmptyLineBeforeAccessModifier: LogicalBlock +SpaceInEmptyBlock: false +FixNamespaceComments: true +IncludeBlocks: Regroup +IndentCaseLabels: true +IndentPPDirectives: None +NamespaceIndentation: None +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PointerAlignment: Left +ReflowComments: false +SeparateDefinitionBlocks: Always +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++20 +TabWidth: 4 +UseTab: Never \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy index 3d705b4..57a94a6 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' HeaderFilterRegex: '\.h$' WarningsAsErrors: '*' CheckOptions: @@ -9,8 +9,6 @@ CheckOptions: value: CamelCase - key: readability-identifier-naming.TypedefCase value: CamelCase - - key: readability-identifier-naming.TypedefIgnoredRegexp - value: (allocator_type|size_type|key_type|value_type|mapped_type|difference_type|pointer|const_pointer|reference|const_reference|iterator_category|const_iterator|iterator|reverse_iterator|const_reverse_iterator|key_compare) - key: readability-identifier-naming.TypeAliasCase value: CamelCase - key: readability-identifier-naming.PrivateMemberSuffix diff --git a/.gitignore b/.gitignore index e05a69a..cafd016 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,6 @@ compile_commands.json .cache* .vscode* build* -libs/eigen + diff --git a/CMakeLists.txt b/CMakeLists.txt index d6bd644..b3fc8f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,51 +6,20 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - include_directories( - ${CMAKE_SOURCE_DIR}/libs/eigen + ${CMAKE_SOURCE_DIR}/libs/Eigen ${CMAKE_SOURCE_DIR}/libs/EigenRand ${CMAKE_SOURCE_DIR}/libs/googletest/googletest/include ${CMAKE_SOURCE_DIR}/libs/mnist/include ${CMAKE_SOURCE_DIR}/src ) - add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) -# add_subdirectory(libs/mnist) - -# add_subdirectory(${CMAKE_SOURCE_DIR}/libs/EigenRand) -# add_subdirectory(${CMAKE_SOURCE_DIR}/libs/eigen) - -add_executable(test_activation_function - tests/test_activation_function.cpp - src/activation_function.cpp -) -target_link_libraries(test_activation_function PRIVATE gtest gtest_main) - -add_executable(test_loss_function - tests/test_loss_function.cpp - src/loss_function.cpp -) -target_link_libraries(test_loss_function PRIVATE gtest gtest_main) - -add_executable(test_layer - tests/test_layer.cpp - src/layer.cpp - src/activation_function.cpp) -target_link_libraries(test_layer PRIVATE gtest gtest_main) - - - -add_executable(test_mnist tests/test_mnist.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp tests/mnist_utils.cpp) +add_executable(test_mnist tests/test_mnist.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp src/random_params.cpp tests/mnist_utils.cpp) target_link_libraries(test_mnist PRIVATE gtest gtest_main) - enable_testing() -add_test(NAME test_activation_function COMMAND test_activation_function) -add_test(NAME test_loss_function COMMAND test_loss_function) -add_test(NAME test_layer COMMAND test_layer) add_test(NAME test_mnist COMMAND test_mnist) diff --git a/README.md b/README.md index a4328c6..deb647e 100644 --- a/README.md +++ b/README.md @@ -1 +1,97 @@ -Это README для *нейросети с 0* \ No newline at end of file +# Нейросеть с нуля +В этом проекте реализованы все основные компоненты нейросети на языке c++. +## Функции активации +Проект поддерживает 5 функций активации: +1. ```ActivationFunc::Name::Id``` - Id +2. ```ActivationFunc::Name::ReLU``` - ReLU +3. ```ActivationFunc::Name::Softmax``` - Softmax +4. ```ActivationFunc::Name::Sigmoid``` - Sigmoid +5. ```ActivationFunc::Name::Tanh``` - Tanh +## Функции потерь +Проект поддерживает 3 функции потерь: +1. ```LossFunc::Name::Mae``` - Mean Absolute Error. +2. ```LossFunc::Name::Mse``` - Mean Squared Error. +3. ```LossFunc::Name::CrossEntropy``` - Cross‑Entropy. +## Можно генерировать случайные параметры слоев самому +Для этого необходимо создать объект класса ```RandomParams``` +1. ```GenerateNormalMatrix``` - генерирует матрицу с нормальным распределением (по умолчанию ~ N(0, 1)); +2. ```GenerateUniformMatrix``` - генерирует матрицу с равномерным распределением(по умолчанию с Uni(0, 1)); +3. ```GenerateConstantMatrix``` - генерирует матрицу, все элементы которой заполнены одним и тем же числом (по умолчанию все элементы равны 0); +Есть также соответствующие аналоги для векторов (например,```GenerateNormalVector```). +## FileReader/FileWriter - отвечают за запись и чтение нейросети с бинарного файла (см. пример ниже) + +## Процесс использование библиотеки +Взаимодействия пользователя с нейросетью можно разделить на 4 части: +построение нейросети, подготовка данных, обучение и тестирование. +```cpp +#include +#include "src/net.h" +using namespace network; + +int main() { + // создание нейросети из 2х слоев + std::vector layer_sizes{784, 256, 10}; + + // создаем свои параметры для слоев (необязательно) + RandomParams rnd(42); + LayerParams first_layer; + LayerParams second_layer; + first_layer.weights = rnd.GenerateNormalMatrix(256, 784, 2, 10); // ~N(2, 10); + first_layer.bias = rnd.GenerateConstantVector(256, 0.01); + second_layer.weights = rnd.GenerateUniformMatrix(10, 256, -1, 1); // ~Uni(-1, 1); + second_layer.bias = rnd.GenerateNormalVector(10); // ~ N(0, 1); + std::vector layer_params = {first_layer, second_layer}; + + std::vector activation_functions{ + ActivationFunc::Name::ReLU, ActivationFunc::Name::Softmax}; // Функции активации на слоях + Net my_model(layer_sizes, activation_functions, layer_params); + + // подготовка данных + Data train_data; + Data test_data; + // чтение базы данных + // ... + + DataLoader data_loader(std::move(train_data)); + LossFunc::Name loss_func = LossFunc::Name::CrossEntropy; + Index batch_size = 70; + Index num_epochs = 17; + + // Параметры для алгоритма оптимизации Адам + DataType learning_rate = 3e-4; + DataType beta1 = 0.9; + DataType beta2 = 0.99; + Info print_info = Info::On; // хотим, чтобы при обучении выводилась информация в терминал + + // обучение + my_model.Train(data_loader, loss_func, batch_size, num_epochs, print_info, learning_rate, beta1, + beta2); + + // Вычисляем результат на тестовой выборки + Matrix prediction = my_model.Evaluate(test_data.input); + + // сравниваем prediction по какому-то правилу с test.output + // ... + + ////////////////////////////////////////////////////////////////// + + //хотим сохранить нейросеть + FileWriter w("net.bin"); + w << my_model; + + // хотим выгрузить нейросеть + Net read_model; + FileReader r("net.bin"); + r >> read_model; + // read_model == my_model +} + +``` + + + + + + + + diff --git a/src/activation_function.cpp b/src/activation_function.cpp index ddcae62..3bbe445 100644 --- a/src/activation_function.cpp +++ b/src/activation_function.cpp @@ -1,56 +1,62 @@ - -#include "activation_function.h" #include -#include "file_reader_writer.h" +#include "activation_function.h" + namespace network { namespace details { struct Sigmoid { static Vector Apply(const Vector& x) { - return x.unaryExpr([](double x) { return 1.0 / (1.0 + std::exp(-x)); }); + return x.unaryExpr([](DataType x) { return 1.0 / (1.0 + std::exp(-x)); }); } + static Matrix GetDifferential(const Vector& x) { Vector applied_sigmoid = Apply(x); - Vector differential = applied_sigmoid.unaryExpr([](double x) { return (1.0 - x) * x; }); + Vector differential = applied_sigmoid.unaryExpr([](DataType x) { return (1.0 - x) * x; }); return differential.asDiagonal(); } }; struct Tanh { static Vector Apply(const Vector& x) { - return x.unaryExpr([](double x) { return std::tanh(x); }); + return x.unaryExpr([](DataType x) { return std::tanh(x); }); } + static Matrix GetDifferential(const Vector& x) { Vector applied_tanh = Apply(x); - Vector differential = applied_tanh.unaryExpr([](double x) { return 1.0 - x * x; }); + Vector differential = applied_tanh.unaryExpr([](DataType x) { return 1.0 - x * x; }); return differential.asDiagonal(); } }; struct ReLU { static Vector Apply(const Vector& x) { - return x.unaryExpr([](double x) { return std::max(0.0, x); }); + return x.unaryExpr([](DataType x) { return std::max(0.0, x); }); } + static Matrix GetDifferential(const Vector& x) { - Vector differential = x.unaryExpr([](double x) { return x > 0.0 ? 1.0 : 0.0; }); + Vector differential = x.unaryExpr([](DataType x) { return x > 0.0 ? 1.0 : 0.0; }); return differential.asDiagonal(); } }; + struct Softmax { static Vector Apply(const Vector& x) { - Vector tmp = x.unaryExpr([](double x) { return std::exp(x); }); + Vector tmp = x.unaryExpr([](DataType x) { return std::exp(x); }); Vector applied_softmax = tmp / tmp.sum(); return applied_softmax; } + static Matrix GetDifferential(const Vector& x) { Vector applied_softmax = Apply(x); Matrix tmp = applied_softmax.asDiagonal(); return tmp - applied_softmax * applied_softmax.transpose(); } }; + struct Id { static Vector Apply(const Vector& x) { return x; } + static Matrix GetDifferential(const Vector& x) { return Eigen::MatrixXd::Identity(x.rows(), x.rows()); } @@ -61,27 +67,27 @@ struct Id { ActivationFunc::ActivationFunc(ActivationFunc::Name name) { switch (name) { case ActivationFunc::Name::Sigmoid: - func_id_ = static_cast(ActivationFunc::Name::Sigmoid); + func_id_ = static_cast(ActivationFunc::Name::Sigmoid); apply_ = details::Sigmoid::Apply; differential_ = details::Sigmoid::GetDifferential; break; case ActivationFunc::Name::ReLU: - func_id_ = static_cast(ActivationFunc::Name::ReLU); + func_id_ = static_cast(ActivationFunc::Name::ReLU); apply_ = details::ReLU::Apply; differential_ = details::ReLU::GetDifferential; break; case ActivationFunc::Name::Tanh: - func_id_ = static_cast(ActivationFunc::Name::Tanh); + func_id_ = static_cast(ActivationFunc::Name::Tanh); apply_ = details::Tanh::Apply; differential_ = details::Tanh::GetDifferential; break; case ActivationFunc::Name::Softmax: - func_id_ = static_cast(ActivationFunc::Name::Softmax); + func_id_ = static_cast(ActivationFunc::Name::Softmax); apply_ = details::Softmax::Apply; differential_ = details::Softmax::GetDifferential; break; case ActivationFunc::Name::Id: - func_id_ = static_cast(ActivationFunc::Name::Id); + func_id_ = static_cast(ActivationFunc::Name::Id); apply_ = details::Id::Apply; differential_ = details::Id::GetDifferential; break; @@ -94,18 +100,21 @@ ActivationFunc::ActivationFunc(ActivationFunc::Name name) { Matrix ActivationFunc::Apply(const Matrix& x) const { Matrix res(x.rows(), x.cols()); - for (int i = 0; i < x.cols(); ++i) { + for (Index i = 0; i < x.cols(); ++i) { res.col(i) = apply_(x.col(i)); } return res; } + Matrix ActivationFunc::GetDifferential(const Vector& x) const { return differential_(x); } -int ActivationFunc::GetFuncId() const { + +Index ActivationFunc::GetFuncId() const { return func_id_; } + bool ActivationFunc::operator==(const ActivationFunc& other) const { return GetFuncId() == other.GetFuncId(); } diff --git a/src/activation_function.h b/src/activation_function.h index 627c793..36c5b58 100644 --- a/src/activation_function.h +++ b/src/activation_function.h @@ -1,8 +1,7 @@ #pragma once #include #include -#include "linalg.h" -#include "file_reader_writer.h" +#include "global_usings.h" namespace network { class ActivationFunc { @@ -15,13 +14,13 @@ class ActivationFunc { explicit ActivationFunc(Name name); Matrix Apply(const Matrix& x) const; Matrix GetDifferential(const Vector& x) const; - int GetFuncId() const; + Index GetFuncId() const; bool operator==(const ActivationFunc& other) const; private: Function apply_; Differential differential_; - int func_id_; + Index func_id_; }; } // namespace network diff --git a/src/adam_optimizer.cpp b/src/adam_optimizer.cpp index eacb3e2..f3bfa4a 100644 --- a/src/adam_optimizer.cpp +++ b/src/adam_optimizer.cpp @@ -3,10 +3,10 @@ namespace network { -AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, double start_learning_rate, double beta1, - double beta2) +AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, DataType learning_rate, DataType beta1, + DataType beta2) : t_(0), - learning_rate_(start_learning_rate), + learning_rate_(learning_rate), beta1_(beta1), beta2_(beta2), moments_{Matrix::Zero(static_cast(rows), static_cast(cols)), @@ -14,24 +14,26 @@ AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, double start_learning_rate, d Vector::Zero(static_cast(rows)), Vector::Zero(static_cast(rows))} { } -void AdamOptimizer::GetCorrection(Matrix& weights_gradient, Vector& bias_gradient) { - assert(weights_gradient.rows() == moments_.weights_m_t.rows() && "invalid weights_gradient"); - assert(weights_gradient.cols() == moments_.weights_m_t.cols() && "invalid weights_gradient"); - assert(bias_gradient.size() == moments_.bias_m_t.size() && "invalid bias_gradient"); +LayerParams AdamOptimizer::GetCorrection(const LayerParams& gradient) { + assert(gradient.weights.rows() == moments_.weights_m_t.rows() && "invalid weights_gradient"); + assert(gradient.weights.cols() == moments_.weights_m_t.cols() && "invalid weights_gradient"); + assert(gradient.bias.size() == moments_.bias_m_t.size() && "invalid bias_gradient"); t_ += 1; - moments_.weights_m_t = beta1_ * moments_.weights_m_t + (1 - beta1_) * weights_gradient; - moments_.bias_m_t = beta1_ * moments_.bias_m_t + (1 - beta1_) * bias_gradient; + moments_.weights_m_t = beta1_ * moments_.weights_m_t + (1 - beta1_) * gradient.weights; + moments_.bias_m_t = beta1_ * moments_.bias_m_t + (1 - beta1_) * gradient.bias; - Matrix squared_weights_gradient = weights_gradient.cwiseProduct(weights_gradient); - Vector squared_bias_gradient = bias_gradient.cwiseProduct(bias_gradient); + Matrix squared_weights_gradient = gradient.weights.cwiseProduct(gradient.weights); + Vector squared_bias_gradient = gradient.bias.cwiseProduct(gradient.bias); moments_.weights_v_t = beta2_ * moments_.weights_v_t + (1 - beta2_) * squared_weights_gradient; moments_.bias_v_t = beta2_ * moments_.bias_v_t + (1 - beta2_) * squared_bias_gradient; - double m_correction = 1.0 - std::pow(beta1_, t_); - double v_correction = 1.0 - std::pow(beta2_, t_); - weights_gradient = learning_rate_ * (moments_.weights_m_t / m_correction).array() / - ((moments_.weights_v_t / v_correction).array().sqrt() + kEps); - bias_gradient = learning_rate_ * (moments_.bias_m_t / m_correction).array() / - ((moments_.bias_v_t / v_correction).array().sqrt() + kEps); + DataType m_correction = 1.0 - std::pow(beta1_, t_); + DataType v_correction = 1.0 - std::pow(beta2_, t_); + Matrix weights_correction = learning_rate_ * (moments_.weights_m_t / m_correction).array() / + ((moments_.weights_v_t / v_correction).array().sqrt() + kEps); + Matrix bias_correction = learning_rate_ * (moments_.bias_m_t / m_correction).array() / + ((moments_.bias_v_t / v_correction).array().sqrt() + kEps); + LayerParams correction{weights_correction, bias_correction}; + return correction; } } // namespace network diff --git a/src/adam_optimizer.h b/src/adam_optimizer.h index 308b981..267b5e4 100644 --- a/src/adam_optimizer.h +++ b/src/adam_optimizer.h @@ -1,11 +1,17 @@ #pragma once -#include "linalg.h" +#include "global_usings.h" namespace network { enum class Rows : Index; enum class Cols : Index; class AdamOptimizer { +public: + AdamOptimizer(Rows rows, Cols cols, DataType learning_rate = kDefaultStartLearningRate, + DataType beta1 = kDefaultBeta1, DataType beta2 = kDefaultBeta2); + LayerParams GetCorrection(const LayerParams& gradient); + +private: struct Moments { Matrix weights_m_t; Matrix weights_v_t; @@ -13,20 +19,14 @@ class AdamOptimizer { Vector bias_v_t; }; -public: - AdamOptimizer(Rows rows, Cols cols, double start_learning_rate = kDefaultStartLearningRate, - double beta1 = kDefaultBeta1, double beta2 = kDefaultBeta2); - void GetCorrection(Matrix& weights_gradient, Vector& bias_gradient); - -private: - static constexpr double kDefaultStartLearningRate = 3e-4; - static constexpr double kDefaultBeta1 = 0.9; - static constexpr double kDefaultBeta2 = 0.99; - static constexpr double kEps = 1e-8; - int t_; - double learning_rate_; - double beta1_; - double beta2_; + static constexpr DataType kDefaultStartLearningRate = 3e-4; + static constexpr DataType kDefaultBeta1 = 0.9; + static constexpr DataType kDefaultBeta2 = 0.99; + static constexpr DataType kEps = 1e-8; + Index t_; + DataType learning_rate_; + DataType beta1_; + DataType beta2_; Moments moments_; }; } // namespace network \ No newline at end of file diff --git a/src/dataloader.cpp b/src/dataloader.cpp index ea1e756..3ef699e 100644 --- a/src/dataloader.cpp +++ b/src/dataloader.cpp @@ -1,32 +1,20 @@ -#include "dataloader.h" + #include #include #include #include -#include -#include +#include "dataloader.h" namespace network { -namespace details { -Shuffle::Shuffle() : gen_(kDefaultSeed) { -} -Shuffle::Shuffle(int seed) : gen_(seed) { -} -void Shuffle::ShuffleData(Index begin, Index end, Data& data) { - assert(begin >= 0 && "Negative start index"); - assert(end > begin && "Invalid range"); - assert(end <= data.input.cols() && "Range exceeds matrix columns"); - assert(data.input.cols() == data.output.cols() && "Matrix size mismatch"); +namespace { +using RandomGenerator = std::mt19937_64; - for (Index i = end - 1; i > begin; --i) { - std::uniform_int_distribution uni(begin, i); - Index j = uni(gen_); - data.input.col(i).swap(data.input.col(j)); - data.output.col(i).swap(data.output.col(j)); - } +RandomGenerator& GetGenerator() { + static constexpr Index kDefaultSeed = 42; + static RandomGenerator gen(kDefaultSeed); + return gen; } - -} // namespace details +} // namespace DataLoader::DataLoader(Data&& data) { assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); @@ -37,46 +25,48 @@ DataLoader::DataLoader(const Data& data) { assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); data_ = data; } -int DataLoader::Size() const { + +Index DataLoader::Size() const { return data_.input.cols(); } -std::vector DataLoader::Batches(int batch_size) const { +std::vector DataLoader::Batches(Index batch_size) const { assert(batch_size > 0 && "Batch size must be positive"); assert(batch_size <= Size() && "Batch size exceeds training data size"); - auto start = std::chrono::high_resolution_clock::now(); - std::vector batches; - int data_size = Size(); - for (int i = 0; i < data_size; i += batch_size) { - int cur_batch_size = std::min(batch_size, data_size - i); - batches.push_back(GetBatch(i, cur_batch_size)); + std::vector batches; + Index data_size = Size(); + batches.reserve(data_size); + for (Index i = 0; i < data_size; i += batch_size) { + Index cur_batch_size = std::min(batch_size, data_size - i); + batches.push_back(std::move(GetBatch(i, cur_batch_size))); } - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - std::cout << "batches time " << duration.count() << std::endl; return batches; } // namespace -void DataLoader::ShuffleData(Shuffle& rnd) { - rnd.ShuffleData(0, Size(), data_); -} -DataLoader::Shuffle& DataLoader::GlobalShuffle() { - static Shuffle rnd; - return rnd; +void DataLoader::ShuffleData() { + RandomGenerator rand_gen = GetGenerator(); + for (Index i = Size() - 1; i > 0; --i) { + std::uniform_int_distribution uni_gen(0, i); + Index j = uni_gen(rand_gen); + data_.input.col(i).swap(data_.input.col(j)); + data_.output.col(i).swap(data_.output.col(j)); + } } + Data DataLoader::GetData() const { return data_; } -Data DataLoader::GetBatch(Index begin, int size) const { +DataView DataLoader::GetBatch(Index begin, Index size) const { assert(begin >= 0 && size > 0 && "Invalid batch parameters"); assert(begin + size <= Size() && "Batch exceeds training data"); - Data batch; - batch.input = data_.input.middleCols(begin, size); - batch.output = data_.output.middleCols(begin, size); + MatrixView inputs = data_.input.middleCols(begin, size); + MatrixView outputs = data_.output.middleCols(begin, size); + DataView batch{inputs, outputs}; + return batch; } diff --git a/src/dataloader.h b/src/dataloader.h index e60ef53..33d40eb 100644 --- a/src/dataloader.h +++ b/src/dataloader.h @@ -1,39 +1,24 @@ #pragma once -#include "linalg.h" +#include "global_usings.h" + namespace network { -struct Data { - Matrix input; - Matrix output; +struct DataView { + MatrixView input; + MatrixView output; }; -namespace details { - -class Shuffle { - -public: - Shuffle(); - Shuffle(int seed); - void ShuffleData(Index begin, Index end, Data& data); -private: - static constexpr int kDefaultSeed = 42; - std::mt19937 gen_; -}; -} // namespace details class DataLoader { - using Shuffle = details::Shuffle; - public: DataLoader(const Data& data); DataLoader(Data&& data); - int Size() const; - std::vector Batches(int batch_size) const; - void ShuffleData(Shuffle& rnd = GlobalShuffle()); + Index Size() const; + std::vector Batches(Index batch_size) const; + void ShuffleData(); Data GetData() const; private: - static Shuffle& GlobalShuffle(); - Data GetBatch(Index begin, int size) const; + DataView GetBatch(Index begin, Index size) const; Data data_; }; diff --git a/src/file_reader_writer.h b/src/file_reader_writer.h index 3ede44f..4105044 100644 --- a/src/file_reader_writer.h +++ b/src/file_reader_writer.h @@ -2,23 +2,24 @@ #pragma once #include #include - -#include "linalg.h" +#include "global_usings.h" namespace network { class FileWriter { + using Writer = std::ofstream; + public: - explicit FileWriter(const std::string& filename) : file_(filename, std::ios::binary) { + explicit FileWriter(const std::string& filename) : w_(filename, std::ios::binary) { } - template + template typename std::enable_if_t, FileWriter&> operator<<(const T& value) { - file_.write(reinterpret_cast(&value), sizeof(T)); + w_.write(reinterpret_cast(&value), sizeof(T)); return *this; } - template + template FileWriter& operator<<(const std::vector& vec) { size_t size = vec.size(); *this << size; @@ -31,38 +32,41 @@ class FileWriter { FileWriter& operator<<(const Matrix& mat) { Matrix::Index rows = mat.rows(); Matrix::Index cols = mat.cols(); - file_.write(reinterpret_cast(&rows), sizeof(Index)); - file_.write(reinterpret_cast(&cols), sizeof(Index)); - file_.write(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + w_.write(reinterpret_cast(&rows), sizeof(Index)); + w_.write(reinterpret_cast(&cols), sizeof(Index)); + w_.write(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); return *this; } FileWriter& operator<<(const Vector& vec) { Vector::Index size = vec.size(); - file_.write(reinterpret_cast(&size), sizeof(Index)); - file_.write(reinterpret_cast(vec.data()), size * sizeof(double)); + w_.write(reinterpret_cast(&size), sizeof(Index)); + w_.write(reinterpret_cast(vec.data()), size * sizeof(double)); return *this; } + void CloseFile() { - file_.close(); + w_.close(); } private: - std::ofstream file_; + Writer w_; }; class FileReader { + using Reader = std::ifstream; + public: - explicit FileReader(const std::string& filename) : file_(filename, std::ios::binary) { + explicit FileReader(const std::string& filename) : r_(filename, std::ios::binary) { } - template + template typename std::enable_if_t, FileReader&> operator>>(T& value) { - file_.read(reinterpret_cast(&value), sizeof(T)); + r_.read(reinterpret_cast(&value), sizeof(T)); return *this; } - template + template FileReader& operator>>(std::vector& vec) { vec.clear(); size_t size; @@ -70,7 +74,6 @@ class FileReader { vec.reserve(size); T element; for (size_t i = 0; i < size; ++i) { - *this >> element; vec.push_back(std::move(element)); } @@ -80,26 +83,27 @@ class FileReader { FileReader& operator>>(Matrix& mat) { Index rows, cols; - file_.read(reinterpret_cast(&rows), sizeof(Index)); - file_.read(reinterpret_cast(&cols), sizeof(Index)); + r_.read(reinterpret_cast(&rows), sizeof(Index)); + r_.read(reinterpret_cast(&cols), sizeof(Index)); mat.resize(rows, cols); - file_.read(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + r_.read(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); return *this; } FileReader& operator>>(Vector& vec) { Index size; - file_.read(reinterpret_cast(&size), sizeof(Index)); + r_.read(reinterpret_cast(&size), sizeof(Index)); vec.resize(size); - file_.read(reinterpret_cast(vec.data()), size * sizeof(double)); + r_.read(reinterpret_cast(vec.data()), size * sizeof(double)); return *this; } + void CloseFile() { - file_.close(); + r_.close(); } private: - std::ifstream file_; + Reader r_; }; } // namespace network \ No newline at end of file diff --git a/src/linalg.h b/src/global_usings.h similarity index 54% rename from src/linalg.h rename to src/global_usings.h index 158babb..856898b 100644 --- a/src/linalg.h +++ b/src/global_usings.h @@ -7,5 +7,17 @@ using Matrix = Eigen::MatrixXd; using Vector = Eigen::VectorXd; using VectorT = Eigen::RowVectorXd; using Index = Eigen::Index; +using MatrixView = Eigen::Ref; +using DataType = Eigen::MatrixXd::Scalar; + +struct LayerParams { + Matrix weights; + Vector bias; +}; + +struct Data { + Matrix input; + Matrix output; +}; } // namespace network \ No newline at end of file diff --git a/src/layer.cpp b/src/layer.cpp index 11a064e..28c20ee 100644 --- a/src/layer.cpp +++ b/src/layer.cpp @@ -1,9 +1,9 @@ #include "layer.h" -#include "EigenRand/Dists/Basic.h" #include "activation_function.h" #include "file_reader_writer.h" #include #include + namespace network { namespace { RandomParams& GetRandomGenerator() { @@ -12,35 +12,14 @@ RandomParams& GetRandomGenerator() { } } // namespace -RandomParams::RandomParams(int seed) : generator_(seed) { -} - -Matrix RandomParams::GenerateNormalMatrix(Index rows, Index cols, double mean, double stdev) { - return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); -} -Vector RandomParams::GenerateNormalVector(Index rows, double mean, double stdev) { - return GenerateNormalMatrix(rows, 1, mean, stdev); -} -Matrix RandomParams::GenerateUniformMatrix(Index rows, Index cols, double min, double max) { - return Eigen::Rand::uniformReal(rows, cols, generator_, min, max); -} -Vector RandomParams::GenerateUniformVector(Index rows, double min, double max) { - return GenerateUniformMatrix(rows, 1, min, max); -} - -Matrix RandomParams::GenerateConstantMatrix(Index rows, Index cols, double value) { - return Eigen::MatrixXd::Constant(rows, cols, value); -} -Vector RandomParams::GenerateConstantVector(Index rows, double value) { - return GenerateConstantMatrix(rows, 1, value); -} - Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name) : func_(name) { InitializeParametrs(static_cast(output_size), static_cast(input_size), name); } + Layer::Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) : weights_(weights), bias_(bias), func_(name) { } + Matrix Layer::ApplyLinear(const Matrix& input_batch) const { assert(weights_.cols() == input_batch.rows() && "can not multiply input_batch on weights matrix"); @@ -58,6 +37,7 @@ void Layer::UpdateWeights(const Matrix& correction) { "invalid correction"); weights_ -= correction; } + void Layer::UpdateBias(const Vector& correction) { assert(correction.rows() == bias_.rows() && "invalid correction"); bias_ -= correction; @@ -69,7 +49,7 @@ Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); Matrix applied_linear = ApplyLinear(input_batch); Matrix tmp = gradient; - for (int i = 0; i < tmp.rows(); ++i) { + for (Index i = 0; i < tmp.rows(); ++i) { Matrix act_func_differ = func_.GetDifferential(applied_linear.col(i)); tmp.row(i) = tmp.row(i) * act_func_differ; } @@ -78,14 +58,14 @@ Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const return new_gradient; } -ParamsGrad Layer::GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const { +LayerParams Layer::GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const { assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); - ParamsGrad grad; + LayerParams grad; Matrix applied_linear = ApplyLinear(input_batch); Matrix matrix_grad_biases(bias_.rows(), input_batch.cols()); - for (int i = 0; i < input_batch.cols(); ++i) { + for (Index i = 0; i < input_batch.cols(); ++i) { matrix_grad_biases.col(i) = (gradient.row(i) * func_.GetDifferential(applied_linear.col(i))).transpose(); } @@ -102,7 +82,7 @@ FileWriter& operator<<(FileWriter& in, const Layer& layer) { } FileReader& operator>>(FileReader& out, Layer& layer) { - int id; + Index id; Matrix weights; Vector bias; out >> id; @@ -111,20 +91,22 @@ FileReader& operator>>(FileReader& out, Layer& layer) { layer = Layer(weights, bias, static_cast(id)); return out; } + Index Layer::GetWeightsRows() const { return weights_.rows(); } + Index Layer::GetWeightsCols() const { return weights_.cols(); } void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name) { RandomParams rnd = GetRandomGenerator(); - constexpr double kConst = 0.01; - double stdev; + constexpr DataType kConst = 0.01; + DataType stdev; switch (name) { case ActivationFunc::Name::ReLU: - stdev = std::sqrt(2.0 / static_cast(cols)); + stdev = std::sqrt(2.0 / static_cast(cols)); weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); bias_ = rnd.GenerateConstantVector(rows, kConst); break; @@ -135,7 +117,7 @@ void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name nam break; default: - stdev = std::sqrt(2.0 / (static_cast(cols) + static_cast(rows))); + stdev = std::sqrt(2.0 / (static_cast(cols) + static_cast(rows))); weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); bias_ = rnd.GenerateConstantVector(rows, 0); break; diff --git a/src/layer.h b/src/layer.h index b95c7da..ee77026 100644 --- a/src/layer.h +++ b/src/layer.h @@ -1,36 +1,16 @@ #pragma once -#include "linalg.h" +#include "global_usings.h" #include "activation_function.h" #include "file_reader_writer.h" -namespace network { - -struct ParamsGrad { - Matrix weights; - Vector bias; -}; - -class RandomParams { - using Generator = Eigen::Rand::Vmt19937_64; - -public: - RandomParams(int seed = kDefaultSeed); - Matrix GenerateNormalMatrix(Index rows, Index cols, double mean = 0, double stdev = 1); - Vector GenerateNormalVector(Index rows, double mean = 0.0, double stdev = 1.0); - Matrix GenerateUniformMatrix(Index rows, Index cols, double low = 0.0, double high = 1.0); - Vector GenerateUniformVector(Index rows, double low = 0.0, double high = 0.0); - Matrix GenerateConstantMatrix(Index rows, Index cols, double value); - Vector GenerateConstantVector(Index rows, double value); +#include "random_params.h" -private: - static constexpr int kDefaultSeed = 42; - Generator generator_{kDefaultSeed}; -}; +namespace network { enum class In : Index; enum class Out : Index; -class Layer { +class Layer { public: Layer() = default; Layer(In input_size, Out output_size, ActivationFunc::Name name); @@ -40,7 +20,7 @@ class Layer { Matrix ApplyLinear(const Matrix& input) const; Matrix Forward(const Matrix& input) const; Matrix Backward(const Matrix& input_batch, const Matrix& gradient) const; - ParamsGrad GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const; + LayerParams GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const; void UpdateWeights(const Matrix& correction); void UpdateBias(const Vector& correction); diff --git a/src/loss_function.cpp b/src/loss_function.cpp index c853965..2aef389 100644 --- a/src/loss_function.cpp +++ b/src/loss_function.cpp @@ -2,35 +2,39 @@ namespace network { namespace details { +struct Mae { + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).cwiseAbs().sum(); + } -struct Mse { - struct Mae { - static double GetValue(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "Mse GetValue"); - return (y_out - y_expected).cwiseAbs().sum(); - } - static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { - assert(y_out.size() == y_expected.size() && "Mse GetGradient"); - return (y_out - y_expected).unaryExpr([](double x) { return x > 0 ? 1.0 : -1.0; }); - } - }; + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return (y_out - y_expected).unaryExpr([](DataType x) { return x > 0 ? 1.0 : -1.0; }); + } +}; - static double GetValue(const Vector &y_out, const Vector &y_expected) { +struct Mse { + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { assert(y_out.size() == y_expected.size() && "Mse GetValue"); return (y_out - y_expected).squaredNorm(); } + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { assert(y_out.size() == y_expected.size() && "Mse GetGradient"); return 2.0 * (y_out - y_expected); } }; + struct CrossEntropy { - static constexpr double kEPS = 1e-12; - static double GetValue(const Vector &y_out, const Vector &y_expected) { + static constexpr DataType kEPS = 1e-12; + + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); return -(y_expected.array() * (y_out.array() + kEPS).log()).sum(); } + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); @@ -39,8 +43,13 @@ struct CrossEntropy { }; } // namespace details + LossFunc::LossFunc(Name name) { switch (name) { + case Name::Mae: + loss_func_ = details::Mae::GetValue; + get_grad_ = details::Mae::GetGradient; + break; case Name::Mse: loss_func_ = details::Mse::GetValue; get_grad_ = details::Mse::GetGradient; @@ -53,20 +62,22 @@ LossFunc::LossFunc(Name name) { assert(false && "Problen in LossFunc constructor"); } } -double LossFunc::Dist(const Vector &y_out, const Vector &y_expected) { + +DataType LossFunc::Apply(const Vector &y_out, const Vector &y_expected) const { return loss_func_(y_out, y_expected); } -Matrix LossFunc::GetGradient(const Matrix &y_out, const Matrix &y_expected) { +Matrix LossFunc::GetGradient(const Matrix &y_out, const Matrix &y_expected) const { assert(y_out.cols() == y_expected.cols() && y_out.rows() == y_expected.rows() && "Matrices have different sizes. GetGradient."); Matrix res(y_out.cols(), y_out.rows()); - for (int i = 0; i < res.rows(); ++i) { + for (Index i = 0; i < res.rows(); ++i) { res.row(i) = GetVectorGrad(y_out.col(i), y_expected.col(i)); } return res; } -VectorT LossFunc::GetVectorGrad(const Vector &y_out, const Vector &y_expected) { + +VectorT LossFunc::GetVectorGrad(const Vector &y_out, const Vector &y_expected) const { return get_grad_(y_out, y_expected).transpose(); } diff --git a/src/loss_function.h b/src/loss_function.h index 9042e82..bdbb7f9 100644 --- a/src/loss_function.h +++ b/src/loss_function.h @@ -1,19 +1,23 @@ #pragma once -#include "linalg.h" +#include "global_usings.h" #include + namespace network { class LossFunc { + using Function = std::function; + using Differential = std::function; + public: enum class Name { Mae, Mse, CrossEntropy }; LossFunc(Name name); - double Dist(const Vector &y_out, const Vector &y_expected); - Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected); + DataType Apply(const Vector &y_out, const Vector &y_expected) const; + Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) const; private: - VectorT GetVectorGrad(const Vector &y_out, const Vector &y_expected); - std::function loss_func_; - std::function get_grad_; + VectorT GetVectorGrad(const Vector &y_out, const Vector &y_expected) const; + Function loss_func_; + Differential get_grad_; }; } // namespace network \ No newline at end of file diff --git a/src/net.cpp b/src/net.cpp index 59799bc..7dee9f7 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -5,6 +5,7 @@ #include "net.h" #include "activation_function.h" #include "adam_optimizer.h" +#include "dataloader.h" #include "layer.h" namespace network { @@ -15,20 +16,22 @@ Net::Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_fu assert(layer_params.empty() || activation_functions.size() == layer_params.size() && "invalid sizes of layer_params or activation_functions"); if (layer_params.empty()) { - for (int i = 1; i < layer_sizes.size(); ++i) { + for (Index i = 1; i < layer_sizes.size(); ++i) { AddLayer(In{layer_sizes[i - 1]}, Out{layer_sizes[i]}, activation_functions[i - 1]); } } else { CheckParams(layer_sizes, activation_functions, layer_params); - for (int i = 0; i < layer_params.size(); ++i) { + for (Index i = 0; i < layer_params.size(); ++i) { AddLayer(layer_params[i].weights, layer_params[i].bias, activation_functions[i]); } } } -Net::ComputedBatches Net::Forward(Matrix& cur_input) const { - std::vector inputs; +Net::ComputedBatches Net::Forward(const Matrix& input) const { + Matrix cur_input = input; + ComputedBatches inputs; + inputs.reserve(layers_.size()); inputs.push_back(cur_input); for (const Layer& layer : layers_) { cur_input = layer.Forward(cur_input); @@ -45,26 +48,25 @@ Matrix Net::Evaluate(const Matrix& batch) const { return cur_batch; } -void Net::Train(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, Info info, - double start_learning_rate, double beta1, double beta2) { +void Net::Train(DataLoader& dl, LossFunc::Name name, Index batch_size, Index num_epochs, Info info, + DataType learning_rate, DataType beta1, DataType beta2) { assert(!layers_.empty() && "empty layers"); auto start = std::chrono::high_resolution_clock::now(); - std::vector optimizers; + Optimizers optimizers; LossFunc loss_func(name); - for (int i = 0; i < layers_.size(); ++i) { + for (Index i = 0; i < layers_.size(); ++i) { Index rows = layers_[i].GetWeightsRows(); Index cols = layers_[i].GetWeightsCols(); - optimizers.emplace_back(Rows{rows}, Cols{cols}, start_learning_rate, beta1, beta2); + optimizers.emplace_back(Rows{rows}, Cols{cols}, learning_rate, beta1, beta2); } if (info == Info::On) { - - for (int epoch = 0; epoch < num_epochs; ++epoch) { + for (Index epoch = 0; epoch < num_epochs; ++epoch) { auto start = std::chrono::high_resolution_clock::now(); std::cout << "Epoch" << " " << epoch + 1 << std::endl; dl.ShuffleData(); - for (Data& train_batch : dl.Batches(batch_size)) { + for (const DataView& train_batch : dl.Batches(batch_size)) { TrainBatch(train_batch, optimizers, loss_func); } auto stop = std::chrono::high_resolution_clock::now(); @@ -81,19 +83,20 @@ void Net::Train(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epo } return; } - for (int epoch = 0; epoch < num_epochs; ++epoch) { + for (Index epoch = 0; epoch < num_epochs; ++epoch) { dl.ShuffleData(); - for (Data& train_batch : dl.Batches(batch_size)) { + for (const DataView& train_batch : dl.Batches(batch_size)) { TrainBatch(train_batch, optimizers, loss_func); } } } + FileWriter& operator<<(FileWriter& out, const Net& net) { out << net.layers_; return out; } -FileReader& operator>>(FileReader& in, Net& net) { +FileReader& operator>>(FileReader& in, Net& net) { in >> net.layers_; return in; } @@ -103,6 +106,7 @@ void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { "incorrect layer"); layers_.emplace_back(input_size, output_size, name); } + void Net::AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) { assert(weights.cols() == bias.rows() && "bad params"); layers_.emplace_back(weights, bias, name); @@ -111,26 +115,27 @@ void Net::AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Na void Net::CheckParams(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, const Params& layer_params) { assert(layer_params.size() == activation_functions.size() && "wrong paramers"); - for (int i = 0; i < layer_sizes.size() - 1; ++i) { + for (Index i = 0; i < layer_sizes.size() - 1; ++i) { assert(layer_sizes[i] == layer_params[i].weights.cols() && "wrong params"); assert(layer_sizes[i + 1] == layer_params[i].weights.rows() && "wrong params"); assert(layer_sizes[i + 1] == layer_params[i].bias.size()); } } -void Net::TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func) { +void Net::TrainBatch(const DataView& data, std::vector& optimizers, + const LossFunc& loss_func) { assert(!layers_.empty() && "no layers"); assert(layers_.size() == optimizers.size() && "different sizes of optimizers and layers"); ComputedBatches computed_batces = Forward(data.input); assert(computed_batces.size() - layers_.size() == 1 && "invalide size of layers_ or computed_batches"); Matrix cur_gradient = loss_func.GetGradient(computed_batces.back(), data.output); - for (int i = layers_.size() - 1; i >= 0; --i) { - ParamsGrad grad = layers_[i].GetParametrsGradient(computed_batces[i], cur_gradient); - optimizers[i].GetCorrection(grad.weights, grad.bias); + for (Index i = layers_.size() - 1; i >= 0; --i) { + LayerParams params_grad = layers_[i].GetParametrsGradient(computed_batces[i], cur_gradient); + LayerParams correction = optimizers[i].GetCorrection(params_grad); cur_gradient = layers_[i].Backward(computed_batces[i], cur_gradient); - layers_[i].UpdateWeights(grad.weights); - layers_[i].UpdateBias(grad.bias); + layers_[i].UpdateWeights(params_grad.weights); + layers_[i].UpdateBias(params_grad.bias); } } } // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h index b5c70b5..faf260e 100644 --- a/src/net.h +++ b/src/net.h @@ -1,7 +1,7 @@ #pragma once #include #include "file_reader_writer.h" -#include "linalg.h" +#include "global_usings.h" #include "activation_function.h" #include "layer.h" #include "loss_function.h" @@ -9,55 +9,40 @@ #include "adam_optimizer.h" namespace network { -namespace details { -struct OptimizersParams { - AdamOptimizer weights; - AdamOptimizer bias; -}; -} // namespace details -struct LayerParams { - Matrix weights; - Vector bias; -}; enum class Info { On, Off }; class Net { - // using Layers = std::vector; - // using ComputedBatches = std::vector; - // using Optimizers = std::vector; - // using LayerSizes = std::vector; - // using ActivationFunctions = std::vector; - -public: using Layers = std::vector; using ComputedBatches = std::vector; - using Optimizers = std::vector; + using Optimizers = std::vector; using LayerSizes = std::vector; using ActivationFunctions = std::vector; using Params = std::vector; + +public: Net() = default; Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, const Params& layer_params = {}); Matrix Evaluate(const Matrix& batch) const; - void Train(DataLoader& dl, LossFunc::Name name, int batch_size, int num_epochs, Info info, - double start_learning_rate = kDefaultStartLearningRate, double beta1 = kDefaultBeta1, - double beta2 = kDefaultBeta2); + void Train(DataLoader& dl, LossFunc::Name name, Index batch_size, Index num_epochs, Info info, + DataType learning_rate = kDefaultStartLearningRate, DataType beta1 = kDefaultBeta1, + DataType beta2 = kDefaultBeta2); friend FileWriter& operator<<(FileWriter& out, const Net& net); friend FileReader& operator>>(FileReader& in, Net& net); bool operator==(const Net& other) const = default; private: - ComputedBatches Forward(Matrix& batch) const; + ComputedBatches Forward(const Matrix& batch) const; void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); void AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); void CheckParams(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, const Params& layer_params); - void TrainBatch(Data& data, std::vector& optimizers, LossFunc& loss_func); - static constexpr double kDefaultStartLearningRate = 3e-4; - static constexpr double kDefaultBeta1 = 0.9; - static constexpr double kDefaultBeta2 = 0.99; + void TrainBatch(const DataView& data, Optimizers& optimizers, const LossFunc& loss_func); + static constexpr DataType kDefaultStartLearningRate = 3e-4; + static constexpr DataType kDefaultBeta1 = 0.9; + static constexpr DataType kDefaultBeta2 = 0.99; Layers layers_; }; diff --git a/src/random_params.cpp b/src/random_params.cpp new file mode 100644 index 0000000..44b4056 --- /dev/null +++ b/src/random_params.cpp @@ -0,0 +1,31 @@ +#include "random_params.h" +#include "global_usings.h" + +namespace network { +RandomParams::RandomParams(Index seed) : generator_(seed) { +} + +Matrix RandomParams::GenerateNormalMatrix(Index rows, Index cols, DataType mean, DataType stdev) { + return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); +} + +Vector RandomParams::GenerateNormalVector(Index rows, DataType mean, DataType stdev) { + return GenerateNormalMatrix(rows, 1, mean, stdev); +} + +Matrix RandomParams::GenerateUniformMatrix(Index rows, Index cols, DataType min, DataType max) { + return Eigen::Rand::uniformReal(rows, cols, generator_, min, max); +} + +Vector RandomParams::GenerateUniformVector(Index rows, DataType min, DataType max) { + return GenerateUniformMatrix(rows, 1, min, max); +} + +Matrix RandomParams::GenerateConstantMatrix(Index rows, Index cols, DataType value) { + return Eigen::MatrixXd::Constant(rows, cols, value); +} + +Vector RandomParams::GenerateConstantVector(Index rows, DataType value) { + return GenerateConstantMatrix(rows, 1, value); +} +} // namespace network \ No newline at end of file diff --git a/src/random_params.h b/src/random_params.h new file mode 100644 index 0000000..f4a6a4e --- /dev/null +++ b/src/random_params.h @@ -0,0 +1,22 @@ +#pragma once +#include "global_usings.h" + +namespace network { + +class RandomParams { + using Generator = Eigen::Rand::Vmt19937_64; + +public: + RandomParams(Index seed = kDefaultSeed); + Matrix GenerateNormalMatrix(Index rows, Index cols, DataType mean = 0, DataType stdev = 1); + Vector GenerateNormalVector(Index rows, DataType mean = 0.0, DataType stdev = 1.0); + Matrix GenerateUniformMatrix(Index rows, Index cols, DataType low = 0.0, DataType high = 1.0); + Vector GenerateUniformVector(Index rows, DataType low = 0.0, DataType high = 0.0); + Matrix GenerateConstantMatrix(Index rows, Index cols, DataType value); + Vector GenerateConstantVector(Index rows, DataType value); + +private: + static constexpr Index kDefaultSeed = 42; + Generator generator_{kDefaultSeed}; +}; +} // namespace network \ No newline at end of file diff --git a/tests/mnist_utils.cpp b/tests/mnist_utils.cpp index 6ca0492..3e7de58 100644 --- a/tests/mnist_utils.cpp +++ b/tests/mnist_utils.cpp @@ -13,6 +13,7 @@ Data MnistUtils::GetTrainData() { Matrix train_labels = LabelToMatrix(dataset.training_labels); return {train_images, train_labels}; } + Data MnistUtils::GetTestData() { auto dataset = mnist::read_dataset(path_); Matrix test_images = ConvertToMatrix(dataset.test_images); @@ -21,12 +22,12 @@ Data MnistUtils::GetTestData() { } Matrix MnistUtils::ConvertToMatrix(const std::vector>& images) { - size_t num_images = images.size(); + Index num_images = images.size(); Matrix eigen_images(784, num_images); - for (size_t i = 0; i < num_images; ++i) { - for (int j = 0; j < 784; ++j) { - eigen_images(j, i) = static_cast(images[i][j]) / 255.0; + for (Index i = 0; i < num_images; ++i) { + for (Index j = 0; j < kInputVectorSize; ++j) { + eigen_images(j, i) = static_cast(images[i][j]) / 255.0; } } @@ -34,34 +35,35 @@ Matrix MnistUtils::ConvertToMatrix(const std::vector>& imag } Matrix MnistUtils::LabelToMatrix(const std::vector& labels) { - size_t num_labels = labels.size(); - Matrix onehot(10, num_labels); + Index num_labels = labels.size(); + Matrix onehot(kOutputVectorSize, num_labels); onehot.setZero(); - for (size_t i = 0; i < num_labels; ++i) { + for (Index i = 0; i < num_labels; ++i) { onehot(labels[i], i) = 1.0; } return onehot; } + double MnistUtils::ComputeAccuracy(const Net& net) { Data test_data = GetTestData(); Matrix net_ans = net.Evaluate(test_data.input); assert(net_ans.cols() == test_data.output.cols() && "wrong cols"); assert(net_ans.rows() == test_data.output.rows() && "wrong rows"); - int test_size = test_data.input.cols(); + Index test_size = test_data.input.cols(); assert(test_size != 0 && "bad test data"); - int cnt = 0; + Index cnt = 0; Index cur_true_ans; Index cur_ans; - for (int i = 0; i < test_size; ++i) { + for (Index i = 0; i < test_size; ++i) { net_ans.col(i).maxCoeff(&cur_ans); test_data.output.col(i).maxCoeff(&cur_true_ans); if (cur_ans == cur_true_ans) { ++cnt; } } - double accuracy = static_cast(cnt) / static_cast(test_size); + DataType accuracy = static_cast(cnt) / static_cast(test_size); std::cout << "Accuracy:" << " " << accuracy << std::endl; return accuracy; } diff --git a/tests/mnist_utils.h b/tests/mnist_utils.h index 0ea7d13..8c177ce 100644 --- a/tests/mnist_utils.h +++ b/tests/mnist_utils.h @@ -2,24 +2,23 @@ #include "mnist/mnist_reader.hpp" namespace network { -struct TrainTestData{ - Data train; - Data test; -}; + class MnistUtils { using Images = std::vector>; using Labels = std::vector; + public: -MnistUtils(const std::string& path); + MnistUtils(const std::string& path); Data GetTrainData(); Data GetTestData(); - double ComputeAccuracy(const Net& net); + DataType ComputeAccuracy(const Net& net); private: Matrix ConvertToMatrix(const Images& images); Matrix LabelToMatrix(const Labels& labels); + static constexpr Index kOutputVectorSize = 10; + static constexpr Index kInputVectorSize = 784; std::string path_; }; - } // namespace network \ No newline at end of file diff --git a/tests/test_activation_function.cpp b/tests/test_activation_function.cpp deleted file mode 100644 index 5fa37d6..0000000 --- a/tests/test_activation_function.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include "linalg.h" -#include "activation_function.h" - -namespace network { - -TEST(ActivationfuncTest, Relu) { - ActivationFunc act(ActivationFunc::Name::ReLU); - Matrix a(2, 2); - a << 1, -1, 2, -2; - Matrix expected_apply(2, 2); - expected_apply << 1, 0, 2, 0; - EXPECT_EQ(expected_apply, act.Apply(a)); - Matrix expected_diff(2, 2); - expected_diff << 1, 0, 0, 1; - EXPECT_EQ(act.GetDifferential(a.col(0)), expected_diff); -} -} // namespace network -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/tests/test_dataloader.cpp b/tests/test_dataloader.cpp deleted file mode 100644 index e210121..0000000 --- a/tests/test_dataloader.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include "dataloader.h" - - -TEST(DataLoder, Load){ - - - - -} - - - - -int main(int argc, char** argv){ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/tests/test_layer.cpp b/tests/test_layer.cpp deleted file mode 100644 index c27acec..0000000 --- a/tests/test_layer.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include "activation_function.h" -#include "linalg.h" -#include "layer.h" - -namespace network { -namespace tests_layer { - -Matrix weights_matrix(3, 2); -Vector bias(3); -Matrix input(2, 3); -Matrix expected_linear_output(3, 3); -Matrix expected_forward_matrix(3, 3); -Matrix expected_backward_matrix(3, 2); -Vector expected_grad_b(3); -Matrix expected_grad_a(3, 2); -Matrix gradient(3, 3); -void SetTestParamers() { - weights_matrix << 6, 1, 1, 1, 10, 7; - bias << 1, 1, 1; - input << 1, -2, 3, -4, 5, -5; - expected_linear_output << 3, -6, 14, -2, 4, -1, -17, 16, -4; - expected_forward_matrix << 3, 0, 14, 0, 4, 0, 0, 16, 0; - gradient << 8, 10, 1, 7, 6, 6, -9, -5, 1; - ///////////// - expected_backward_matrix << 48, 8, 66, 48, -54, -9; - expected_grad_b << -1, 6, 6; - expected_grad_a << -19, 13, -12, 30, -12, 30; - expected_grad_b /= 3; - expected_grad_a /= 3; -} - -TEST(Correction, Forward) { - Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); - Matrix output = layer.ApplyLinear(input); - EXPECT_EQ(output, expected_linear_output); - Matrix forward_matrix = layer.Forward(input); - EXPECT_EQ(forward_matrix, expected_forward_matrix); -} -TEST(Correction, Backward) { - Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); - Matrix bacward_matrix = layer.Backward(input, gradient); - EXPECT_EQ(expected_backward_matrix, bacward_matrix); -} -TEST(Correction, Gradients) { - Layer layer(weights_matrix, bias, ActivationFunc::Name::ReLU); - WeightsBiasGradient grad_weights_bias = layer.GetWeightsBiasGradient(input, gradient); - EXPECT_EQ(expected_grad_a, grad_weights_bias.weights); - EXPECT_EQ(expected_grad_b, grad_weights_bias.bias); -} -// namespace network -} // namespace tests_layer -} // namespace network -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/test_loss_function.cpp b/tests/test_loss_function.cpp deleted file mode 100644 index 7e294ad..0000000 --- a/tests/test_loss_function.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include -#include "linalg.h" \ No newline at end of file diff --git a/tests/test_mnist.cpp b/tests/test_mnist.cpp index 7c14249..f5c503f 100644 --- a/tests/test_mnist.cpp +++ b/tests/test_mnist.cpp @@ -5,6 +5,7 @@ #include "file_reader_writer.h" #include "loss_function.h" #include "mnist_utils.h" + void ClearFile(std::string filename) { std::ofstream file(filename, std::ios::binary | std::ios::trunc); file.close(); @@ -20,11 +21,11 @@ TEST(Mnist, WriteRead) { MnistUtils mnist_utils(path); Data train_data = mnist_utils.GetTrainData(); DataLoader dl(std::move(train_data)); - constexpr int kNumEpocs = 17; - constexpr int kBatchSize = 64; + constexpr Index kNumEpocs = 17; + constexpr Index kBatchSize = 64; nn.Train(dl, LossFunc::Name::CrossEntropy, kBatchSize, kNumEpocs, Info::On); - constexpr double kExpectedAccuracy = 0; - double accuracy = mnist_utils.ComputeAccuracy(nn); + constexpr DataType kExpectedAccuracy = 0; + DataType accuracy = mnist_utils.ComputeAccuracy(nn); EXPECT_GE(accuracy, kExpectedAccuracy); std::string filename = "save_net.bin"; From e2cb2669c583546a616030235871fcec16e7d8d8 Mon Sep 17 00:00:00 2001 From: Demjan Klimov Date: Sun, 20 Apr 2025 14:38:39 +0300 Subject: [PATCH 13/13] make tests better --- CMakeLists.txt | 4 +- src/file_reader_writer.h | 29 ++++++++- src/net.cpp | 3 +- tests/mnist_utils.cpp | 7 +-- tests/mnist_utils.h | 7 ++- tests/test_mnist.cpp | 47 --------------- tests/test_net.cpp | 123 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 161 insertions(+), 59 deletions(-) delete mode 100644 tests/test_mnist.cpp create mode 100644 tests/test_net.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b3fc8f8..0ba8532 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ include_directories( add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) -add_executable(test_mnist tests/test_mnist.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp src/random_params.cpp tests/mnist_utils.cpp) -target_link_libraries(test_mnist PRIVATE gtest gtest_main) +add_executable(test_net tests/test_net.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp src/random_params.cpp tests/mnist_utils.cpp) +target_link_libraries(test_net PRIVATE gtest gtest_main) enable_testing() add_test(NAME test_mnist COMMAND test_mnist) diff --git a/src/file_reader_writer.h b/src/file_reader_writer.h index 4105044..690a599 100644 --- a/src/file_reader_writer.h +++ b/src/file_reader_writer.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include "global_usings.h" @@ -8,9 +9,13 @@ namespace network { class FileWriter { using Writer = std::ofstream; + using Path = std::filesystem::path; public: - explicit FileWriter(const std::string& filename) : w_(filename, std::ios::binary) { + explicit FileWriter(const Path& path) { + assert(IsValidPath(path) && "invalid path"); + w_ = std::move(Writer(path, std::ios::binary)); + assert(IsOpen() && "file did not open"); } template @@ -50,14 +55,26 @@ class FileWriter { } private: + bool IsValidPath(const Path& path) { + return std::filesystem::exists(path); + } + + bool IsOpen() { + return w_.is_open(); + } + Writer w_; }; class FileReader { using Reader = std::ifstream; + using Path = std::filesystem::path; public: - explicit FileReader(const std::string& filename) : r_(filename, std::ios::binary) { + explicit FileReader(const Path& path) { + assert(IsValidPath(path) && "invalid path"); + r_ = std::move(Reader(path, std::ios::binary)); + assert(IsOpen() && "file did not open"); } template @@ -103,6 +120,14 @@ class FileReader { } private: + bool IsValidPath(const Path& path) { + return std::filesystem::exists(path); + } + + bool IsOpen() { + return r_.is_open(); + } + Reader r_; }; diff --git a/src/net.cpp b/src/net.cpp index 7dee9f7..e9b1c56 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -9,6 +9,7 @@ #include "layer.h" namespace network { + Net::Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, const Params& layer_params) { assert(!layer_sizes.empty() && "layer_sizes empty"); @@ -108,7 +109,7 @@ void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { } void Net::AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) { - assert(weights.cols() == bias.rows() && "bad params"); + assert(weights.rows() == bias.rows() && "bad params"); layers_.emplace_back(weights, bias, name); } diff --git a/tests/mnist_utils.cpp b/tests/mnist_utils.cpp index 3e7de58..5c64411 100644 --- a/tests/mnist_utils.cpp +++ b/tests/mnist_utils.cpp @@ -1,9 +1,9 @@ #include "net.h" -#include #include "mnist_utils.h" +#include "mnist/mnist_reader.hpp" namespace network { -MnistUtils::MnistUtils(const std::string& path) : path_(path) { +MnistUtils::MnistUtils(const Path& path) : path_(path) { } Data MnistUtils::GetTrainData() { @@ -46,7 +46,7 @@ Matrix MnistUtils::LabelToMatrix(const std::vector& labels) { return onehot; } -double MnistUtils::ComputeAccuracy(const Net& net) { +DataType MnistUtils::ComputeAccuracy(const Net& net) { Data test_data = GetTestData(); Matrix net_ans = net.Evaluate(test_data.input); assert(net_ans.cols() == test_data.output.cols() && "wrong cols"); @@ -64,7 +64,6 @@ double MnistUtils::ComputeAccuracy(const Net& net) { } } DataType accuracy = static_cast(cnt) / static_cast(test_size); - std::cout << "Accuracy:" << " " << accuracy << std::endl; return accuracy; } } // namespace network diff --git a/tests/mnist_utils.h b/tests/mnist_utils.h index 8c177ce..989bb21 100644 --- a/tests/mnist_utils.h +++ b/tests/mnist_utils.h @@ -1,14 +1,15 @@ #include "net.h" -#include "mnist/mnist_reader.hpp" + namespace network { class MnistUtils { using Images = std::vector>; using Labels = std::vector; + using Path = std::filesystem::path; public: - MnistUtils(const std::string& path); + MnistUtils(const Path& path); Data GetTrainData(); Data GetTestData(); DataType ComputeAccuracy(const Net& net); @@ -18,7 +19,7 @@ class MnistUtils { Matrix LabelToMatrix(const Labels& labels); static constexpr Index kOutputVectorSize = 10; static constexpr Index kInputVectorSize = 784; - std::string path_; + Path path_; }; } // namespace network \ No newline at end of file diff --git a/tests/test_mnist.cpp b/tests/test_mnist.cpp deleted file mode 100644 index f5c503f..0000000 --- a/tests/test_mnist.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include -#include "activation_function.h" -#include "dataloader.h" -#include "file_reader_writer.h" -#include "loss_function.h" -#include "mnist_utils.h" - -void ClearFile(std::string filename) { - std::ofstream file(filename, std::ios::binary | std::ios::trunc); - file.close(); -} - -TEST(Mnist, WriteRead) { - using namespace network; - std::vector layer_sizes = {784, 256, 10}; - std::vector activation_functions = {ActivationFunc::Name::ReLU, - ActivationFunc::Name::Softmax}; - Net nn(layer_sizes, activation_functions); - std::string path = "../libs/mnist"; - MnistUtils mnist_utils(path); - Data train_data = mnist_utils.GetTrainData(); - DataLoader dl(std::move(train_data)); - constexpr Index kNumEpocs = 17; - constexpr Index kBatchSize = 64; - nn.Train(dl, LossFunc::Name::CrossEntropy, kBatchSize, kNumEpocs, Info::On); - constexpr DataType kExpectedAccuracy = 0; - DataType accuracy = mnist_utils.ComputeAccuracy(nn); - EXPECT_GE(accuracy, kExpectedAccuracy); - - std::string filename = "save_net.bin"; - - ClearFile(filename); - FileWriter w(filename); - w << nn; - w.CloseFile(); - FileReader r(filename); - Net net; - r >> net; - r.CloseFile(); - EXPECT_EQ(nn, net); -} - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/tests/test_net.cpp b/tests/test_net.cpp new file mode 100644 index 0000000..850c47c --- /dev/null +++ b/tests/test_net.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include "activation_function.h" +#include "dataloader.h" +#include "file_reader_writer.h" +#include "global_usings.h" +#include "loss_function.h" +#include "mnist_utils.h" +#include "random_params.h" +using namespace network; + +void ClearFile(std::filesystem::path filename) { + std::ofstream file(filename, std::ios::binary | std::ios::trunc); + file.close(); +} + +Net BuildMnistNet() { + constexpr Index kFirstSize = 784; + constexpr Index kSecondSize = 256; + constexpr Index kThirdSize = 10; + std::vector layer_sizes = {kFirstSize, kSecondSize, kThirdSize}; + std::vector activation_functions = {ActivationFunc::Name::ReLU, + ActivationFunc::Name::Softmax}; + Net nn(layer_sizes, activation_functions); + return nn; +} + +MnistUtils PrepareMnistUtils() { + const std::filesystem::path path = "../libs/mnist"; + MnistUtils mnist_utils(path); + return mnist_utils; +} + +DataLoader PrepareDataloader(MnistUtils& mnist_utils) { + Data train_data = mnist_utils.GetTrainData(); + DataLoader dl(std::move(train_data)); + return dl; +} + +void TrainNet(Net& nn, DataLoader& dl) { + constexpr Index kNumEpocs = 17; + constexpr Index kBatchSize = 64; + constexpr Info kInfo = Info::On; + constexpr LossFunc::Name kLossFunc = LossFunc::Name::CrossEntropy; + nn.Train(dl, kLossFunc, kBatchSize, kNumEpocs, kInfo); +} + +Index GetRandomNum() { + std::mt19937 gen(std::random_device{}()); + static constexpr Index kLow = 1; + static constexpr Index kHigh = 1000; + std::uniform_int_distribution dist(kLow, kHigh); + Index random_number = dist(gen); + return random_number; +} + +ActivationFunc::Name GetRandomActivationFunc() { + static std::vector activation_functions = { + ActivationFunc::Name::Tanh, ActivationFunc::Name::Id, ActivationFunc::Name::ReLU, + ActivationFunc::Name::Sigmoid, ActivationFunc::Name::Softmax}; + Index rand_num = GetRandomNum(); + return activation_functions[rand_num % activation_functions.size()]; +} + +Net BuildRandomNet() { + std::vector sizes(GetRandomNum()); + + for (Index i = 0; i < sizes.size(); ++i) { + sizes[i] = GetRandomNum(); + } + std::vector layer_params; + RandomParams rnd; + for (Index i = 0; i < sizes.size() - 1; ++i) { + Matrix weights = rnd.GenerateUniformMatrix(sizes[i + 1], sizes[i]); + Vector bias = rnd.GenerateNormalVector(sizes[i + 1]); + layer_params.emplace_back(weights, bias); + } + std::vector activation_functions(sizes.size() - 1); + for (Index i = 0; i < activation_functions.size(); ++i) { + activation_functions[i] = GetRandomActivationFunc(); + } + Net nn(sizes, activation_functions, layer_params); + return nn; +} + +TEST(Net, ReadWrite) { + const std::filesystem::path filename = "save_net.bin"; + constexpr Index kIter = 10; + for (Index i = 0; i < kIter; ++i) { + ClearFile(filename); + FileWriter w(filename); + Net nn = BuildRandomNet(); + w << nn; + w.CloseFile(); + FileReader r(filename); + Net net; + r >> net; + r.CloseFile(); + EXPECT_EQ(nn, net); + } + ClearFile(filename); +} + +TEST(Net, MnistCorrection) { + Net nn = BuildMnistNet(); + MnistUtils mnist_utils = PrepareMnistUtils(); + DataLoader dl = PrepareDataloader(mnist_utils); + TrainNet(nn, dl); + constexpr DataType kExpectedAccuracy = 98; + DataType accuracy = mnist_utils.ComputeAccuracy(nn); + EXPECT_GE(accuracy, kExpectedAccuracy) << accuracy << std::endl; +} + +int main(int argc, char** argv) { + try { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); + } catch (std::exception& e) { + std::cerr << e.what() << std::endl; + } catch (...) {} +} \ No newline at end of file