diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..9fcd2d3 --- /dev/null +++ b/.clang-format @@ -0,0 +1,82 @@ +Language: Cpp +BasedOnStyle: Google +IndentWidth: 4 +AccessModifierOffset: -4 +ColumnLimit: 100 +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +DerivePointerAlignment: true +KeepEmptyLinesAtTheStartOfBlocks: false +SortIncludes: Never +MaxEmptyLinesToKeep: 1 +InsertTrailingCommas: None +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: None +AlignOperands: Align +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: false +AllowShortLambdasOnASingleLine: Inline +AlwaysBreakAfterReturnType: None +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BreakBeforeBraces: Attach +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterStruct: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +CompactNamespaces: false +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +EmptyLineBeforeAccessModifier: LogicalBlock +SpaceInEmptyBlock: false +FixNamespaceComments: true +IncludeBlocks: Regroup +IndentCaseLabels: true +IndentPPDirectives: None +NamespaceIndentation: None +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PointerAlignment: Left +ReflowComments: false +SeparateDefinitionBlocks: Always +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++20 +TabWidth: 4 +UseTab: Never \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..57a94a6 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,39 @@ +--- +Checks: '-*,cppcoreguidelines-avoid-goto,cppcoreguidelines-pro-type-const-cast, google-readability-casting, modernize-replace-random-shuffle, readability-braces-around-statements, readability-container-size-empty, readability-redundant-control-flow, readability-redundant-string-init, modernize-use-nullptr, readability-identifier-naming, google-build-using-namespace' +HeaderFilterRegex: '\.h$' +WarningsAsErrors: '*' +CheckOptions: + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.TypedefCase + value: CamelCase + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.PrivateMemberSuffix + value: '_' + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: CamelCase + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.GlobalConstantPrefix + value: k + - key: readability-identifier-naming.GlobalConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: google-runtime-int.TypeSuffix + value: _t \ No newline at end of file diff --git a/.gitignore b/.gitignore index 259148f..cafd016 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,33 @@ -# Prerequisites *.d -# Compiled Object files *.slo *.lo *.o *.obj -# Precompiled Headers *.gch *.pch -# Compiled Dynamic libraries *.so *.dylib *.dll -# Fortran module files *.mod *.smod -# Compiled Static libraries *.lai *.la *.a *.lib -# Executables *.exe *.out *.app + +compile_commands.json +.cache* +.vscode* +build* + + + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a981fa8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,13 @@ + +[submodule "libs/EigenRand"] + path = libs/EigenRand + url = https://github.com/bab2min/EigenRand +[submodule "libs/googletest"] + path = libs/googletest + url = https://github.com/google/googletest.git +[submodule "libs/mnist"] + path = libs/mnist + url = https://github.com/wichtounet/mnist.git +[submodule "libs/Eigen"] + path = libs/Eigen + url = https://gitlab.com/libeigen/eigen.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..0ba8532 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.10) +project(NeuralNetwork LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +include_directories( + ${CMAKE_SOURCE_DIR}/libs/Eigen + ${CMAKE_SOURCE_DIR}/libs/EigenRand + ${CMAKE_SOURCE_DIR}/libs/googletest/googletest/include + ${CMAKE_SOURCE_DIR}/libs/mnist/include + ${CMAKE_SOURCE_DIR}/src +) + +add_subdirectory(${CMAKE_SOURCE_DIR}/libs/googletest) + +add_executable(test_net tests/test_net.cpp src/adam_optimizer.cpp src/net.cpp src/loss_function.cpp src/activation_function.cpp src/dataloader.cpp src/layer.cpp src/random_params.cpp tests/mnist_utils.cpp) +target_link_libraries(test_net PRIVATE gtest gtest_main) + +enable_testing() +add_test(NAME test_mnist COMMAND test_mnist) + + + diff --git a/README.md b/README.md index a4328c6..deb647e 100644 --- a/README.md +++ b/README.md @@ -1 +1,97 @@ -Это README для *нейросети с 0* \ No newline at end of file +# Нейросеть с нуля +В этом проекте реализованы все основные компоненты нейросети на языке c++. +## Функции активации +Проект поддерживает 5 функций активации: +1. ```ActivationFunc::Name::Id``` - Id +2. ```ActivationFunc::Name::ReLU``` - ReLU +3. ```ActivationFunc::Name::Softmax``` - Softmax +4. ```ActivationFunc::Name::Sigmoid``` - Sigmoid +5. ```ActivationFunc::Name::Tanh``` - Tanh +## Функции потерь +Проект поддерживает 3 функции потерь: +1. ```LossFunc::Name::Mae``` - Mean Absolute Error. +2. ```LossFunc::Name::Mse``` - Mean Squared Error. +3. ```LossFunc::Name::CrossEntropy``` - Cross‑Entropy. +## Можно генерировать случайные параметры слоев самому +Для этого необходимо создать объект класса ```RandomParams``` +1. ```GenerateNormalMatrix``` - генерирует матрицу с нормальным распределением (по умолчанию ~ N(0, 1)); +2. ```GenerateUniformMatrix``` - генерирует матрицу с равномерным распределением(по умолчанию с Uni(0, 1)); +3. ```GenerateConstantMatrix``` - генерирует матрицу, все элементы которой заполнены одним и тем же числом (по умолчанию все элементы равны 0); +Есть также соответствующие аналоги для векторов (например,```GenerateNormalVector```). +## FileReader/FileWriter - отвечают за запись и чтение нейросети с бинарного файла (см. пример ниже) + +## Процесс использование библиотеки +Взаимодействия пользователя с нейросетью можно разделить на 4 части: +построение нейросети, подготовка данных, обучение и тестирование. +```cpp +#include +#include "src/net.h" +using namespace network; + +int main() { + // создание нейросети из 2х слоев + std::vector layer_sizes{784, 256, 10}; + + // создаем свои параметры для слоев (необязательно) + RandomParams rnd(42); + LayerParams first_layer; + LayerParams second_layer; + first_layer.weights = rnd.GenerateNormalMatrix(256, 784, 2, 10); // ~N(2, 10); + first_layer.bias = rnd.GenerateConstantVector(256, 0.01); + second_layer.weights = rnd.GenerateUniformMatrix(10, 256, -1, 1); // ~Uni(-1, 1); + second_layer.bias = rnd.GenerateNormalVector(10); // ~ N(0, 1); + std::vector layer_params = {first_layer, second_layer}; + + std::vector activation_functions{ + ActivationFunc::Name::ReLU, ActivationFunc::Name::Softmax}; // Функции активации на слоях + Net my_model(layer_sizes, activation_functions, layer_params); + + // подготовка данных + Data train_data; + Data test_data; + // чтение базы данных + // ... + + DataLoader data_loader(std::move(train_data)); + LossFunc::Name loss_func = LossFunc::Name::CrossEntropy; + Index batch_size = 70; + Index num_epochs = 17; + + // Параметры для алгоритма оптимизации Адам + DataType learning_rate = 3e-4; + DataType beta1 = 0.9; + DataType beta2 = 0.99; + Info print_info = Info::On; // хотим, чтобы при обучении выводилась информация в терминал + + // обучение + my_model.Train(data_loader, loss_func, batch_size, num_epochs, print_info, learning_rate, beta1, + beta2); + + // Вычисляем результат на тестовой выборки + Matrix prediction = my_model.Evaluate(test_data.input); + + // сравниваем prediction по какому-то правилу с test.output + // ... + + ////////////////////////////////////////////////////////////////// + + //хотим сохранить нейросеть + FileWriter w("net.bin"); + w << my_model; + + // хотим выгрузить нейросеть + Net read_model; + FileReader r("net.bin"); + r >> read_model; + // read_model == my_model +} + +``` + + + + + + + + diff --git a/libs/Eigen b/libs/Eigen new file mode 160000 index 0000000..3147391 --- /dev/null +++ b/libs/Eigen @@ -0,0 +1 @@ +Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/libs/EigenRand b/libs/EigenRand new file mode 160000 index 0000000..f3190cd --- /dev/null +++ b/libs/EigenRand @@ -0,0 +1 @@ +Subproject commit f3190cd7f85c5878c949625cb2811262ff050d66 diff --git a/libs/googletest b/libs/googletest new file mode 160000 index 0000000..4902ea2 --- /dev/null +++ b/libs/googletest @@ -0,0 +1 @@ +Subproject commit 4902ea2d7c6faed89b6facee00baa34bb108fc0d diff --git a/libs/mnist b/libs/mnist new file mode 160000 index 0000000..3b65c35 --- /dev/null +++ b/libs/mnist @@ -0,0 +1 @@ +Subproject commit 3b65c35ede53b687376c4302eeb44fdf76e0129b diff --git a/src/activation_function.cpp b/src/activation_function.cpp new file mode 100644 index 0000000..3bbe445 --- /dev/null +++ b/src/activation_function.cpp @@ -0,0 +1,122 @@ +#include +#include "activation_function.h" + +namespace network { +namespace details { +struct Sigmoid { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](DataType x) { return 1.0 / (1.0 + std::exp(-x)); }); + } + + static Matrix GetDifferential(const Vector& x) { + Vector applied_sigmoid = Apply(x); + Vector differential = applied_sigmoid.unaryExpr([](DataType x) { return (1.0 - x) * x; }); + return differential.asDiagonal(); + } +}; + +struct Tanh { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](DataType x) { return std::tanh(x); }); + } + + static Matrix GetDifferential(const Vector& x) { + Vector applied_tanh = Apply(x); + Vector differential = applied_tanh.unaryExpr([](DataType x) { return 1.0 - x * x; }); + return differential.asDiagonal(); + } +}; + +struct ReLU { + static Vector Apply(const Vector& x) { + return x.unaryExpr([](DataType x) { return std::max(0.0, x); }); + } + + static Matrix GetDifferential(const Vector& x) { + Vector differential = x.unaryExpr([](DataType x) { return x > 0.0 ? 1.0 : 0.0; }); + return differential.asDiagonal(); + } +}; + +struct Softmax { + static Vector Apply(const Vector& x) { + Vector tmp = x.unaryExpr([](DataType x) { return std::exp(x); }); + Vector applied_softmax = tmp / tmp.sum(); + return applied_softmax; + } + + static Matrix GetDifferential(const Vector& x) { + Vector applied_softmax = Apply(x); + Matrix tmp = applied_softmax.asDiagonal(); + return tmp - applied_softmax * applied_softmax.transpose(); + } +}; + +struct Id { + static Vector Apply(const Vector& x) { + return x; + } + + static Matrix GetDifferential(const Vector& x) { + return Eigen::MatrixXd::Identity(x.rows(), x.rows()); + } +}; + +} // namespace details + +ActivationFunc::ActivationFunc(ActivationFunc::Name name) { + switch (name) { + case ActivationFunc::Name::Sigmoid: + func_id_ = static_cast(ActivationFunc::Name::Sigmoid); + apply_ = details::Sigmoid::Apply; + differential_ = details::Sigmoid::GetDifferential; + break; + case ActivationFunc::Name::ReLU: + func_id_ = static_cast(ActivationFunc::Name::ReLU); + apply_ = details::ReLU::Apply; + differential_ = details::ReLU::GetDifferential; + break; + case ActivationFunc::Name::Tanh: + func_id_ = static_cast(ActivationFunc::Name::Tanh); + apply_ = details::Tanh::Apply; + differential_ = details::Tanh::GetDifferential; + break; + case ActivationFunc::Name::Softmax: + func_id_ = static_cast(ActivationFunc::Name::Softmax); + apply_ = details::Softmax::Apply; + differential_ = details::Softmax::GetDifferential; + break; + case ActivationFunc::Name::Id: + func_id_ = static_cast(ActivationFunc::Name::Id); + apply_ = details::Id::Apply; + differential_ = details::Id::GetDifferential; + break; + + default: + assert(false && "invalid arguments in ActivationFunc constructor"); + } +} + +Matrix ActivationFunc::Apply(const Matrix& x) const { + Matrix res(x.rows(), x.cols()); + + for (Index i = 0; i < x.cols(); ++i) { + res.col(i) = apply_(x.col(i)); + } + + return res; +} + +Matrix ActivationFunc::GetDifferential(const Vector& x) const { + return differential_(x); +} + +Index ActivationFunc::GetFuncId() const { + return func_id_; +} + +bool ActivationFunc::operator==(const ActivationFunc& other) const { + return GetFuncId() == other.GetFuncId(); +} + +} // namespace network \ No newline at end of file diff --git a/src/activation_function.h b/src/activation_function.h new file mode 100644 index 0000000..36c5b58 --- /dev/null +++ b/src/activation_function.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include "global_usings.h" + +namespace network { +class ActivationFunc { + using Function = std::function; + using Differential = std::function; + +public: + ActivationFunc() = default; + enum class Name { Sigmoid, ReLU, Tanh, Softmax, Id }; + explicit ActivationFunc(Name name); + Matrix Apply(const Matrix& x) const; + Matrix GetDifferential(const Vector& x) const; + Index GetFuncId() const; + bool operator==(const ActivationFunc& other) const; + +private: + Function apply_; + Differential differential_; + Index func_id_; +}; + +} // namespace network diff --git a/src/adam_optimizer.cpp b/src/adam_optimizer.cpp new file mode 100644 index 0000000..f3bfa4a --- /dev/null +++ b/src/adam_optimizer.cpp @@ -0,0 +1,39 @@ +#include "adam_optimizer.h" +#include + +namespace network { + +AdamOptimizer::AdamOptimizer(Rows rows, Cols cols, DataType learning_rate, DataType beta1, + DataType beta2) + : t_(0), + learning_rate_(learning_rate), + beta1_(beta1), + beta2_(beta2), + moments_{Matrix::Zero(static_cast(rows), static_cast(cols)), + Matrix::Zero(static_cast(rows), static_cast(cols)), + Vector::Zero(static_cast(rows)), Vector::Zero(static_cast(rows))} { +} + +LayerParams AdamOptimizer::GetCorrection(const LayerParams& gradient) { + assert(gradient.weights.rows() == moments_.weights_m_t.rows() && "invalid weights_gradient"); + assert(gradient.weights.cols() == moments_.weights_m_t.cols() && "invalid weights_gradient"); + assert(gradient.bias.size() == moments_.bias_m_t.size() && "invalid bias_gradient"); + t_ += 1; + moments_.weights_m_t = beta1_ * moments_.weights_m_t + (1 - beta1_) * gradient.weights; + moments_.bias_m_t = beta1_ * moments_.bias_m_t + (1 - beta1_) * gradient.bias; + + Matrix squared_weights_gradient = gradient.weights.cwiseProduct(gradient.weights); + Vector squared_bias_gradient = gradient.bias.cwiseProduct(gradient.bias); + moments_.weights_v_t = beta2_ * moments_.weights_v_t + (1 - beta2_) * squared_weights_gradient; + moments_.bias_v_t = beta2_ * moments_.bias_v_t + (1 - beta2_) * squared_bias_gradient; + DataType m_correction = 1.0 - std::pow(beta1_, t_); + DataType v_correction = 1.0 - std::pow(beta2_, t_); + Matrix weights_correction = learning_rate_ * (moments_.weights_m_t / m_correction).array() / + ((moments_.weights_v_t / v_correction).array().sqrt() + kEps); + Matrix bias_correction = learning_rate_ * (moments_.bias_m_t / m_correction).array() / + ((moments_.bias_v_t / v_correction).array().sqrt() + kEps); + LayerParams correction{weights_correction, bias_correction}; + return correction; +} + +} // namespace network diff --git a/src/adam_optimizer.h b/src/adam_optimizer.h new file mode 100644 index 0000000..267b5e4 --- /dev/null +++ b/src/adam_optimizer.h @@ -0,0 +1,32 @@ +#pragma once +#include "global_usings.h" + +namespace network { +enum class Rows : Index; +enum class Cols : Index; + +class AdamOptimizer { +public: + AdamOptimizer(Rows rows, Cols cols, DataType learning_rate = kDefaultStartLearningRate, + DataType beta1 = kDefaultBeta1, DataType beta2 = kDefaultBeta2); + LayerParams GetCorrection(const LayerParams& gradient); + +private: + struct Moments { + Matrix weights_m_t; + Matrix weights_v_t; + Vector bias_m_t; + Vector bias_v_t; + }; + + static constexpr DataType kDefaultStartLearningRate = 3e-4; + static constexpr DataType kDefaultBeta1 = 0.9; + static constexpr DataType kDefaultBeta2 = 0.99; + static constexpr DataType kEps = 1e-8; + Index t_; + DataType learning_rate_; + DataType beta1_; + DataType beta2_; + Moments moments_; +}; +} // namespace network \ No newline at end of file diff --git a/src/dataloader.cpp b/src/dataloader.cpp new file mode 100644 index 0000000..3ef699e --- /dev/null +++ b/src/dataloader.cpp @@ -0,0 +1,73 @@ + +#include +#include +#include +#include +#include "dataloader.h" + +namespace network { +namespace { +using RandomGenerator = std::mt19937_64; + +RandomGenerator& GetGenerator() { + static constexpr Index kDefaultSeed = 42; + static RandomGenerator gen(kDefaultSeed); + return gen; +} +} // namespace + +DataLoader::DataLoader(Data&& data) { + assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); + data_ = std::move(data); +} + +DataLoader::DataLoader(const Data& data) { + assert(data.input.cols() == data.output.cols() && "Data input and output columns mismatch"); + data_ = data; +} + +Index DataLoader::Size() const { + return data_.input.cols(); +} + +std::vector DataLoader::Batches(Index batch_size) const { + assert(batch_size > 0 && "Batch size must be positive"); + assert(batch_size <= Size() && "Batch size exceeds training data size"); + std::vector batches; + Index data_size = Size(); + batches.reserve(data_size); + for (Index i = 0; i < data_size; i += batch_size) { + Index cur_batch_size = std::min(batch_size, data_size - i); + batches.push_back(std::move(GetBatch(i, cur_batch_size))); + } + + return batches; +} + +// namespace + +void DataLoader::ShuffleData() { + RandomGenerator rand_gen = GetGenerator(); + for (Index i = Size() - 1; i > 0; --i) { + std::uniform_int_distribution uni_gen(0, i); + Index j = uni_gen(rand_gen); + data_.input.col(i).swap(data_.input.col(j)); + data_.output.col(i).swap(data_.output.col(j)); + } +} + +Data DataLoader::GetData() const { + return data_; +} + +DataView DataLoader::GetBatch(Index begin, Index size) const { + assert(begin >= 0 && size > 0 && "Invalid batch parameters"); + assert(begin + size <= Size() && "Batch exceeds training data"); + MatrixView inputs = data_.input.middleCols(begin, size); + MatrixView outputs = data_.output.middleCols(begin, size); + DataView batch{inputs, outputs}; + + return batch; +} + +} // namespace network \ No newline at end of file diff --git a/src/dataloader.h b/src/dataloader.h new file mode 100644 index 0000000..33d40eb --- /dev/null +++ b/src/dataloader.h @@ -0,0 +1,25 @@ +#pragma once +#include "global_usings.h" + +namespace network { + +struct DataView { + MatrixView input; + MatrixView output; +}; + +class DataLoader { +public: + DataLoader(const Data& data); + DataLoader(Data&& data); + Index Size() const; + std::vector Batches(Index batch_size) const; + void ShuffleData(); + Data GetData() const; + +private: + DataView GetBatch(Index begin, Index size) const; + Data data_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/file_reader_writer.h b/src/file_reader_writer.h new file mode 100644 index 0000000..690a599 --- /dev/null +++ b/src/file_reader_writer.h @@ -0,0 +1,134 @@ + +#pragma once +#include +#include +#include +#include "global_usings.h" + +namespace network { + +class FileWriter { + using Writer = std::ofstream; + using Path = std::filesystem::path; + +public: + explicit FileWriter(const Path& path) { + assert(IsValidPath(path) && "invalid path"); + w_ = std::move(Writer(path, std::ios::binary)); + assert(IsOpen() && "file did not open"); + } + + template + typename std::enable_if_t, FileWriter&> operator<<(const T& value) { + w_.write(reinterpret_cast(&value), sizeof(T)); + return *this; + } + + template + FileWriter& operator<<(const std::vector& vec) { + size_t size = vec.size(); + *this << size; + for (const auto& element : vec) { + *this << element; + } + return *this; + } + + FileWriter& operator<<(const Matrix& mat) { + Matrix::Index rows = mat.rows(); + Matrix::Index cols = mat.cols(); + w_.write(reinterpret_cast(&rows), sizeof(Index)); + w_.write(reinterpret_cast(&cols), sizeof(Index)); + w_.write(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + return *this; + } + + FileWriter& operator<<(const Vector& vec) { + Vector::Index size = vec.size(); + w_.write(reinterpret_cast(&size), sizeof(Index)); + w_.write(reinterpret_cast(vec.data()), size * sizeof(double)); + return *this; + } + + void CloseFile() { + w_.close(); + } + +private: + bool IsValidPath(const Path& path) { + return std::filesystem::exists(path); + } + + bool IsOpen() { + return w_.is_open(); + } + + Writer w_; +}; + +class FileReader { + using Reader = std::ifstream; + using Path = std::filesystem::path; + +public: + explicit FileReader(const Path& path) { + assert(IsValidPath(path) && "invalid path"); + r_ = std::move(Reader(path, std::ios::binary)); + assert(IsOpen() && "file did not open"); + } + + template + typename std::enable_if_t, FileReader&> operator>>(T& value) { + r_.read(reinterpret_cast(&value), sizeof(T)); + return *this; + } + + template + FileReader& operator>>(std::vector& vec) { + vec.clear(); + size_t size; + *this >> size; + vec.reserve(size); + T element; + for (size_t i = 0; i < size; ++i) { + *this >> element; + vec.push_back(std::move(element)); + } + + return *this; + } + + FileReader& operator>>(Matrix& mat) { + Index rows, cols; + r_.read(reinterpret_cast(&rows), sizeof(Index)); + r_.read(reinterpret_cast(&cols), sizeof(Index)); + mat.resize(rows, cols); + r_.read(reinterpret_cast(mat.data()), rows * cols * sizeof(double)); + return *this; + } + + FileReader& operator>>(Vector& vec) { + Index size; + r_.read(reinterpret_cast(&size), sizeof(Index)); + vec.resize(size); + r_.read(reinterpret_cast(vec.data()), size * sizeof(double)); + return *this; + } + + void CloseFile() { + r_.close(); + } + +private: + bool IsValidPath(const Path& path) { + return std::filesystem::exists(path); + } + + bool IsOpen() { + return r_.is_open(); + } + + Reader r_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/global_usings.h b/src/global_usings.h new file mode 100644 index 0000000..856898b --- /dev/null +++ b/src/global_usings.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include + +namespace network { +using Matrix = Eigen::MatrixXd; +using Vector = Eigen::VectorXd; +using VectorT = Eigen::RowVectorXd; +using Index = Eigen::Index; +using MatrixView = Eigen::Ref; +using DataType = Eigen::MatrixXd::Scalar; + +struct LayerParams { + Matrix weights; + Vector bias; +}; + +struct Data { + Matrix input; + Matrix output; +}; + +} // namespace network \ No newline at end of file diff --git a/src/layer.cpp b/src/layer.cpp new file mode 100644 index 0000000..28c20ee --- /dev/null +++ b/src/layer.cpp @@ -0,0 +1,127 @@ +#include "layer.h" +#include "activation_function.h" +#include "file_reader_writer.h" +#include +#include + +namespace network { +namespace { +RandomParams& GetRandomGenerator() { + static RandomParams rnd; + return rnd; +} +} // namespace + +Layer::Layer(In input_size, Out output_size, ActivationFunc::Name name) : func_(name) { + InitializeParametrs(static_cast(output_size), static_cast(input_size), name); +} + +Layer::Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) + : weights_(weights), bias_(bias), func_(name) { +} + +Matrix Layer::ApplyLinear(const Matrix& input_batch) const { + assert(weights_.cols() == input_batch.rows() && + "can not multiply input_batch on weights matrix"); + Matrix output_batch = weights_ * input_batch; + output_batch.colwise() += bias_; + return output_batch; +} + +Matrix Layer::Forward(const Matrix& input) const { + return func_.Apply(ApplyLinear(input)); +} + +void Layer::UpdateWeights(const Matrix& correction) { + assert(weights_.cols() == correction.cols() && weights_.rows() == correction.rows() && + "invalid correction"); + weights_ -= correction; +} + +void Layer::UpdateBias(const Vector& correction) { + assert(correction.rows() == bias_.rows() && "invalid correction"); + bias_ -= correction; +} + +Matrix Layer::Backward(const Matrix& input_batch, const Matrix& gradient) const { + assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); + assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); + assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); + Matrix applied_linear = ApplyLinear(input_batch); + Matrix tmp = gradient; + for (Index i = 0; i < tmp.rows(); ++i) { + Matrix act_func_differ = func_.GetDifferential(applied_linear.col(i)); + tmp.row(i) = tmp.row(i) * act_func_differ; + } + Matrix new_gradient(input_batch.cols(), input_batch.rows()); + new_gradient = tmp * weights_; + return new_gradient; +} + +LayerParams Layer::GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const { + assert(input_batch.cols() == gradient.rows() && "different size of gradient and input_batch"); + assert(weights_.rows() == gradient.cols() && "wrong size of gradient or of weights matrix"); + assert(weights_.cols() == input_batch.rows() && "wrong size of weight matrix or input_batch"); + LayerParams grad; + Matrix applied_linear = ApplyLinear(input_batch); + Matrix matrix_grad_biases(bias_.rows(), input_batch.cols()); + for (Index i = 0; i < input_batch.cols(); ++i) { + matrix_grad_biases.col(i) = + (gradient.row(i) * func_.GetDifferential(applied_linear.col(i))).transpose(); + } + grad.bias = matrix_grad_biases.rowwise().sum() / matrix_grad_biases.cols(); + grad.weights = matrix_grad_biases * input_batch.transpose() / matrix_grad_biases.cols(); + return grad; +} + +FileWriter& operator<<(FileWriter& in, const Layer& layer) { + in << layer.func_.GetFuncId(); + in << layer.weights_; + in << layer.bias_; + return in; +} + +FileReader& operator>>(FileReader& out, Layer& layer) { + Index id; + Matrix weights; + Vector bias; + out >> id; + out >> weights; + out >> bias; + layer = Layer(weights, bias, static_cast(id)); + return out; +} + +Index Layer::GetWeightsRows() const { + return weights_.rows(); +} + +Index Layer::GetWeightsCols() const { + return weights_.cols(); +} + +void Layer::InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name) { + RandomParams rnd = GetRandomGenerator(); + constexpr DataType kConst = 0.01; + DataType stdev; + switch (name) { + case ActivationFunc::Name::ReLU: + stdev = std::sqrt(2.0 / static_cast(cols)); + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.GenerateConstantVector(rows, kConst); + break; + + case ActivationFunc::Name::Id: + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, kConst); + bias_ = rnd.GenerateConstantVector(rows, 0); + break; + + default: + stdev = std::sqrt(2.0 / (static_cast(cols) + static_cast(rows))); + weights_ = rnd.GenerateNormalMatrix(rows, cols, 0, stdev); + bias_ = rnd.GenerateConstantVector(rows, 0); + break; + } +} + +} // namespace network \ No newline at end of file diff --git a/src/layer.h b/src/layer.h new file mode 100644 index 0000000..ee77026 --- /dev/null +++ b/src/layer.h @@ -0,0 +1,40 @@ + +#pragma once +#include "global_usings.h" +#include "activation_function.h" +#include "file_reader_writer.h" +#include "random_params.h" + +namespace network { + +enum class In : Index; +enum class Out : Index; + +class Layer { +public: + Layer() = default; + Layer(In input_size, Out output_size, ActivationFunc::Name name); + + Layer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); + + Matrix ApplyLinear(const Matrix& input) const; + Matrix Forward(const Matrix& input) const; + Matrix Backward(const Matrix& input_batch, const Matrix& gradient) const; + LayerParams GetParametrsGradient(const Matrix& input_batch, const Matrix& gradient) const; + void UpdateWeights(const Matrix& correction); + void UpdateBias(const Vector& correction); + + friend FileWriter& operator<<(FileWriter& in, const Layer& layer); + friend FileReader& operator>>(FileReader& out, Layer& layer); + bool operator==(const Layer& other) const = default; + Index GetWeightsRows() const; + Index GetWeightsCols() const; + +private: + void InitializeParametrs(Index rows, Index cols, ActivationFunc::Name name); + ActivationFunc func_; + Matrix weights_; + Vector bias_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/loss_function.cpp b/src/loss_function.cpp new file mode 100644 index 0000000..2aef389 --- /dev/null +++ b/src/loss_function.cpp @@ -0,0 +1,84 @@ +#include "loss_function.h" + +namespace network { +namespace details { +struct Mae { + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).cwiseAbs().sum(); + } + + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return (y_out - y_expected).unaryExpr([](DataType x) { return x > 0 ? 1.0 : -1.0; }); + } +}; + +struct Mse { + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetValue"); + return (y_out - y_expected).squaredNorm(); + } + + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "Mse GetGradient"); + return 2.0 * (y_out - y_expected); + } +}; + +struct CrossEntropy { + static constexpr DataType kEPS = 1e-12; + + static DataType GetValue(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "CrossEntropy GetValue"); + + return -(y_expected.array() * (y_out.array() + kEPS).log()).sum(); + } + + static Vector GetGradient(const Vector &y_out, const Vector &y_expected) { + assert(y_out.size() == y_expected.size() && "CrossEnropy GetGradient"); + + return -(y_expected.array() / (y_out.array() + kEPS)); + } +}; + +} // namespace details + +LossFunc::LossFunc(Name name) { + switch (name) { + case Name::Mae: + loss_func_ = details::Mae::GetValue; + get_grad_ = details::Mae::GetGradient; + break; + case Name::Mse: + loss_func_ = details::Mse::GetValue; + get_grad_ = details::Mse::GetGradient; + break; + case Name::CrossEntropy: + loss_func_ = details::CrossEntropy::GetValue; + get_grad_ = details::CrossEntropy::GetGradient; + break; + default: + assert(false && "Problen in LossFunc constructor"); + } +} + +DataType LossFunc::Apply(const Vector &y_out, const Vector &y_expected) const { + return loss_func_(y_out, y_expected); +} + +Matrix LossFunc::GetGradient(const Matrix &y_out, const Matrix &y_expected) const { + assert(y_out.cols() == y_expected.cols() && y_out.rows() == y_expected.rows() && + "Matrices have different sizes. GetGradient."); + Matrix res(y_out.cols(), y_out.rows()); + for (Index i = 0; i < res.rows(); ++i) { + res.row(i) = GetVectorGrad(y_out.col(i), y_expected.col(i)); + } + return res; +} + +VectorT LossFunc::GetVectorGrad(const Vector &y_out, const Vector &y_expected) const { + return get_grad_(y_out, y_expected).transpose(); +} + +} // namespace network diff --git a/src/loss_function.h b/src/loss_function.h new file mode 100644 index 0000000..bdbb7f9 --- /dev/null +++ b/src/loss_function.h @@ -0,0 +1,23 @@ +#pragma once +#include "global_usings.h" +#include + +namespace network { + +class LossFunc { + using Function = std::function; + using Differential = std::function; + +public: + enum class Name { Mae, Mse, CrossEntropy }; + LossFunc(Name name); + DataType Apply(const Vector &y_out, const Vector &y_expected) const; + Matrix GetGradient(const Matrix &y_out, const Matrix &y_expected) const; + +private: + VectorT GetVectorGrad(const Vector &y_out, const Vector &y_expected) const; + Function loss_func_; + Differential get_grad_; +}; + +} // namespace network \ No newline at end of file diff --git a/src/net.cpp b/src/net.cpp new file mode 100644 index 0000000..e9b1c56 --- /dev/null +++ b/src/net.cpp @@ -0,0 +1,142 @@ + +#include +#include +#include +#include "net.h" +#include "activation_function.h" +#include "adam_optimizer.h" +#include "dataloader.h" +#include "layer.h" + +namespace network { + +Net::Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params) { + assert(!layer_sizes.empty() && "layer_sizes empty"); + assert(layer_sizes.size() - activation_functions.size() == 1 && "invalid parametrs"); + assert(layer_params.empty() || activation_functions.size() == layer_params.size() && + "invalid sizes of layer_params or activation_functions"); + if (layer_params.empty()) { + for (Index i = 1; i < layer_sizes.size(); ++i) { + AddLayer(In{layer_sizes[i - 1]}, Out{layer_sizes[i]}, activation_functions[i - 1]); + } + + } else { + CheckParams(layer_sizes, activation_functions, layer_params); + for (Index i = 0; i < layer_params.size(); ++i) { + AddLayer(layer_params[i].weights, layer_params[i].bias, activation_functions[i]); + } + } +} + +Net::ComputedBatches Net::Forward(const Matrix& input) const { + Matrix cur_input = input; + ComputedBatches inputs; + inputs.reserve(layers_.size()); + inputs.push_back(cur_input); + for (const Layer& layer : layers_) { + cur_input = layer.Forward(cur_input); + inputs.push_back(cur_input); + } + return inputs; +} + +Matrix Net::Evaluate(const Matrix& batch) const { + Matrix cur_batch = batch; + for (const Layer& layer : layers_) { + cur_batch = layer.Forward(cur_batch); + } + return cur_batch; +} + +void Net::Train(DataLoader& dl, LossFunc::Name name, Index batch_size, Index num_epochs, Info info, + DataType learning_rate, DataType beta1, DataType beta2) { + assert(!layers_.empty() && "empty layers"); + auto start = std::chrono::high_resolution_clock::now(); + Optimizers optimizers; + LossFunc loss_func(name); + for (Index i = 0; i < layers_.size(); ++i) { + Index rows = layers_[i].GetWeightsRows(); + Index cols = layers_[i].GetWeightsCols(); + optimizers.emplace_back(Rows{rows}, Cols{cols}, learning_rate, beta1, beta2); + } + if (info == Info::On) { + for (Index epoch = 0; epoch < num_epochs; ++epoch) { + auto start = std::chrono::high_resolution_clock::now(); + + std::cout << "Epoch" << " " << epoch + 1 << std::endl; + + dl.ShuffleData(); + for (const DataView& train_batch : dl.Batches(batch_size)) { + TrainBatch(train_batch, optimizers, loss_func); + } + auto stop = std::chrono::high_resolution_clock::now(); + auto time = stop - start; + auto hours = std::chrono::duration_cast(time); + auto minutes = + std::chrono::duration_cast(time % std::chrono::hours(1)); + auto seconds = + std::chrono::duration_cast(time % std::chrono::minutes(1)); + + std::cout << "Epoch time:" << " " << std::setfill('0') << std::setw(2) << hours.count() + << ":" << std::setw(2) << minutes.count() << ":" << std::setw(2) + << seconds.count() << "\n"; + } + return; + } + for (Index epoch = 0; epoch < num_epochs; ++epoch) { + dl.ShuffleData(); + for (const DataView& train_batch : dl.Batches(batch_size)) { + TrainBatch(train_batch, optimizers, loss_func); + } + } +} + +FileWriter& operator<<(FileWriter& out, const Net& net) { + out << net.layers_; + return out; +} + +FileReader& operator>>(FileReader& in, Net& net) { + in >> net.layers_; + return in; +} + +void Net::AddLayer(In input_size, Out output_size, ActivationFunc::Name name) { + assert((layers_.empty() || layers_.back().GetWeightsRows() == static_cast(input_size)) && + "incorrect layer"); + layers_.emplace_back(input_size, output_size, name); +} + +void Net::AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name) { + assert(weights.rows() == bias.rows() && "bad params"); + layers_.emplace_back(weights, bias, name); +} + +void Net::CheckParams(const LayerSizes& layer_sizes, + const ActivationFunctions& activation_functions, const Params& layer_params) { + assert(layer_params.size() == activation_functions.size() && "wrong paramers"); + for (Index i = 0; i < layer_sizes.size() - 1; ++i) { + assert(layer_sizes[i] == layer_params[i].weights.cols() && "wrong params"); + assert(layer_sizes[i + 1] == layer_params[i].weights.rows() && "wrong params"); + assert(layer_sizes[i + 1] == layer_params[i].bias.size()); + } +} + +void Net::TrainBatch(const DataView& data, std::vector& optimizers, + const LossFunc& loss_func) { + assert(!layers_.empty() && "no layers"); + assert(layers_.size() == optimizers.size() && "different sizes of optimizers and layers"); + ComputedBatches computed_batces = Forward(data.input); + assert(computed_batces.size() - layers_.size() == 1 && + "invalide size of layers_ or computed_batches"); + Matrix cur_gradient = loss_func.GetGradient(computed_batces.back(), data.output); + for (Index i = layers_.size() - 1; i >= 0; --i) { + LayerParams params_grad = layers_[i].GetParametrsGradient(computed_batces[i], cur_gradient); + LayerParams correction = optimizers[i].GetCorrection(params_grad); + cur_gradient = layers_[i].Backward(computed_batces[i], cur_gradient); + layers_[i].UpdateWeights(params_grad.weights); + layers_[i].UpdateBias(params_grad.bias); + } +} +} // namespace network \ No newline at end of file diff --git a/src/net.h b/src/net.h new file mode 100644 index 0000000..faf260e --- /dev/null +++ b/src/net.h @@ -0,0 +1,49 @@ +#pragma once +#include +#include "file_reader_writer.h" +#include "global_usings.h" +#include "activation_function.h" +#include "layer.h" +#include "loss_function.h" +#include "dataloader.h" +#include "adam_optimizer.h" + +namespace network { + +enum class Info { On, Off }; + +class Net { + using Layers = std::vector; + using ComputedBatches = std::vector; + using Optimizers = std::vector; + using LayerSizes = std::vector; + using ActivationFunctions = std::vector; + using Params = std::vector; + +public: + Net() = default; + Net(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params = {}); + + Matrix Evaluate(const Matrix& batch) const; + void Train(DataLoader& dl, LossFunc::Name name, Index batch_size, Index num_epochs, Info info, + DataType learning_rate = kDefaultStartLearningRate, DataType beta1 = kDefaultBeta1, + DataType beta2 = kDefaultBeta2); + friend FileWriter& operator<<(FileWriter& out, const Net& net); + friend FileReader& operator>>(FileReader& in, Net& net); + bool operator==(const Net& other) const = default; + +private: + ComputedBatches Forward(const Matrix& batch) const; + void AddLayer(In input_size, Out output_size, ActivationFunc::Name name); + void AddLayer(const Matrix& weights, const Vector& bias, ActivationFunc::Name name); + void CheckParams(const LayerSizes& layer_sizes, const ActivationFunctions& activation_functions, + const Params& layer_params); + void TrainBatch(const DataView& data, Optimizers& optimizers, const LossFunc& loss_func); + static constexpr DataType kDefaultStartLearningRate = 3e-4; + static constexpr DataType kDefaultBeta1 = 0.9; + static constexpr DataType kDefaultBeta2 = 0.99; + + Layers layers_; +}; +} // namespace network \ No newline at end of file diff --git a/src/random_params.cpp b/src/random_params.cpp new file mode 100644 index 0000000..44b4056 --- /dev/null +++ b/src/random_params.cpp @@ -0,0 +1,31 @@ +#include "random_params.h" +#include "global_usings.h" + +namespace network { +RandomParams::RandomParams(Index seed) : generator_(seed) { +} + +Matrix RandomParams::GenerateNormalMatrix(Index rows, Index cols, DataType mean, DataType stdev) { + return Eigen::Rand::normal(rows, cols, generator_, mean, stdev); +} + +Vector RandomParams::GenerateNormalVector(Index rows, DataType mean, DataType stdev) { + return GenerateNormalMatrix(rows, 1, mean, stdev); +} + +Matrix RandomParams::GenerateUniformMatrix(Index rows, Index cols, DataType min, DataType max) { + return Eigen::Rand::uniformReal(rows, cols, generator_, min, max); +} + +Vector RandomParams::GenerateUniformVector(Index rows, DataType min, DataType max) { + return GenerateUniformMatrix(rows, 1, min, max); +} + +Matrix RandomParams::GenerateConstantMatrix(Index rows, Index cols, DataType value) { + return Eigen::MatrixXd::Constant(rows, cols, value); +} + +Vector RandomParams::GenerateConstantVector(Index rows, DataType value) { + return GenerateConstantMatrix(rows, 1, value); +} +} // namespace network \ No newline at end of file diff --git a/src/random_params.h b/src/random_params.h new file mode 100644 index 0000000..f4a6a4e --- /dev/null +++ b/src/random_params.h @@ -0,0 +1,22 @@ +#pragma once +#include "global_usings.h" + +namespace network { + +class RandomParams { + using Generator = Eigen::Rand::Vmt19937_64; + +public: + RandomParams(Index seed = kDefaultSeed); + Matrix GenerateNormalMatrix(Index rows, Index cols, DataType mean = 0, DataType stdev = 1); + Vector GenerateNormalVector(Index rows, DataType mean = 0.0, DataType stdev = 1.0); + Matrix GenerateUniformMatrix(Index rows, Index cols, DataType low = 0.0, DataType high = 1.0); + Vector GenerateUniformVector(Index rows, DataType low = 0.0, DataType high = 0.0); + Matrix GenerateConstantMatrix(Index rows, Index cols, DataType value); + Vector GenerateConstantVector(Index rows, DataType value); + +private: + static constexpr Index kDefaultSeed = 42; + Generator generator_{kDefaultSeed}; +}; +} // namespace network \ No newline at end of file diff --git a/tests/mnist_utils.cpp b/tests/mnist_utils.cpp new file mode 100644 index 0000000..5c64411 --- /dev/null +++ b/tests/mnist_utils.cpp @@ -0,0 +1,69 @@ +#include "net.h" +#include "mnist_utils.h" +#include "mnist/mnist_reader.hpp" + +namespace network { +MnistUtils::MnistUtils(const Path& path) : path_(path) { +} + +Data MnistUtils::GetTrainData() { + auto dataset = mnist::read_dataset(path_); + assert(!dataset.training_images.empty() && "empty dataset"); + Matrix train_images = ConvertToMatrix(dataset.training_images); + Matrix train_labels = LabelToMatrix(dataset.training_labels); + return {train_images, train_labels}; +} + +Data MnistUtils::GetTestData() { + auto dataset = mnist::read_dataset(path_); + Matrix test_images = ConvertToMatrix(dataset.test_images); + Matrix test_labels = LabelToMatrix(dataset.test_labels); + return {test_images, test_labels}; +} + +Matrix MnistUtils::ConvertToMatrix(const std::vector>& images) { + Index num_images = images.size(); + Matrix eigen_images(784, num_images); + + for (Index i = 0; i < num_images; ++i) { + for (Index j = 0; j < kInputVectorSize; ++j) { + eigen_images(j, i) = static_cast(images[i][j]) / 255.0; + } + } + + return eigen_images; +} + +Matrix MnistUtils::LabelToMatrix(const std::vector& labels) { + Index num_labels = labels.size(); + Matrix onehot(kOutputVectorSize, num_labels); + onehot.setZero(); + + for (Index i = 0; i < num_labels; ++i) { + onehot(labels[i], i) = 1.0; + } + + return onehot; +} + +DataType MnistUtils::ComputeAccuracy(const Net& net) { + Data test_data = GetTestData(); + Matrix net_ans = net.Evaluate(test_data.input); + assert(net_ans.cols() == test_data.output.cols() && "wrong cols"); + assert(net_ans.rows() == test_data.output.rows() && "wrong rows"); + Index test_size = test_data.input.cols(); + assert(test_size != 0 && "bad test data"); + Index cnt = 0; + Index cur_true_ans; + Index cur_ans; + for (Index i = 0; i < test_size; ++i) { + net_ans.col(i).maxCoeff(&cur_ans); + test_data.output.col(i).maxCoeff(&cur_true_ans); + if (cur_ans == cur_true_ans) { + ++cnt; + } + } + DataType accuracy = static_cast(cnt) / static_cast(test_size); + return accuracy; +} +} // namespace network diff --git a/tests/mnist_utils.h b/tests/mnist_utils.h new file mode 100644 index 0000000..989bb21 --- /dev/null +++ b/tests/mnist_utils.h @@ -0,0 +1,25 @@ +#include "net.h" + + +namespace network { + +class MnistUtils { + using Images = std::vector>; + using Labels = std::vector; + using Path = std::filesystem::path; + +public: + MnistUtils(const Path& path); + Data GetTrainData(); + Data GetTestData(); + DataType ComputeAccuracy(const Net& net); + +private: + Matrix ConvertToMatrix(const Images& images); + Matrix LabelToMatrix(const Labels& labels); + static constexpr Index kOutputVectorSize = 10; + static constexpr Index kInputVectorSize = 784; + Path path_; +}; + +} // namespace network \ No newline at end of file diff --git a/tests/save_net.bin b/tests/save_net.bin new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_net.cpp b/tests/test_net.cpp new file mode 100644 index 0000000..850c47c --- /dev/null +++ b/tests/test_net.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include "activation_function.h" +#include "dataloader.h" +#include "file_reader_writer.h" +#include "global_usings.h" +#include "loss_function.h" +#include "mnist_utils.h" +#include "random_params.h" +using namespace network; + +void ClearFile(std::filesystem::path filename) { + std::ofstream file(filename, std::ios::binary | std::ios::trunc); + file.close(); +} + +Net BuildMnistNet() { + constexpr Index kFirstSize = 784; + constexpr Index kSecondSize = 256; + constexpr Index kThirdSize = 10; + std::vector layer_sizes = {kFirstSize, kSecondSize, kThirdSize}; + std::vector activation_functions = {ActivationFunc::Name::ReLU, + ActivationFunc::Name::Softmax}; + Net nn(layer_sizes, activation_functions); + return nn; +} + +MnistUtils PrepareMnistUtils() { + const std::filesystem::path path = "../libs/mnist"; + MnistUtils mnist_utils(path); + return mnist_utils; +} + +DataLoader PrepareDataloader(MnistUtils& mnist_utils) { + Data train_data = mnist_utils.GetTrainData(); + DataLoader dl(std::move(train_data)); + return dl; +} + +void TrainNet(Net& nn, DataLoader& dl) { + constexpr Index kNumEpocs = 17; + constexpr Index kBatchSize = 64; + constexpr Info kInfo = Info::On; + constexpr LossFunc::Name kLossFunc = LossFunc::Name::CrossEntropy; + nn.Train(dl, kLossFunc, kBatchSize, kNumEpocs, kInfo); +} + +Index GetRandomNum() { + std::mt19937 gen(std::random_device{}()); + static constexpr Index kLow = 1; + static constexpr Index kHigh = 1000; + std::uniform_int_distribution dist(kLow, kHigh); + Index random_number = dist(gen); + return random_number; +} + +ActivationFunc::Name GetRandomActivationFunc() { + static std::vector activation_functions = { + ActivationFunc::Name::Tanh, ActivationFunc::Name::Id, ActivationFunc::Name::ReLU, + ActivationFunc::Name::Sigmoid, ActivationFunc::Name::Softmax}; + Index rand_num = GetRandomNum(); + return activation_functions[rand_num % activation_functions.size()]; +} + +Net BuildRandomNet() { + std::vector sizes(GetRandomNum()); + + for (Index i = 0; i < sizes.size(); ++i) { + sizes[i] = GetRandomNum(); + } + std::vector layer_params; + RandomParams rnd; + for (Index i = 0; i < sizes.size() - 1; ++i) { + Matrix weights = rnd.GenerateUniformMatrix(sizes[i + 1], sizes[i]); + Vector bias = rnd.GenerateNormalVector(sizes[i + 1]); + layer_params.emplace_back(weights, bias); + } + std::vector activation_functions(sizes.size() - 1); + for (Index i = 0; i < activation_functions.size(); ++i) { + activation_functions[i] = GetRandomActivationFunc(); + } + Net nn(sizes, activation_functions, layer_params); + return nn; +} + +TEST(Net, ReadWrite) { + const std::filesystem::path filename = "save_net.bin"; + constexpr Index kIter = 10; + for (Index i = 0; i < kIter; ++i) { + ClearFile(filename); + FileWriter w(filename); + Net nn = BuildRandomNet(); + w << nn; + w.CloseFile(); + FileReader r(filename); + Net net; + r >> net; + r.CloseFile(); + EXPECT_EQ(nn, net); + } + ClearFile(filename); +} + +TEST(Net, MnistCorrection) { + Net nn = BuildMnistNet(); + MnistUtils mnist_utils = PrepareMnistUtils(); + DataLoader dl = PrepareDataloader(mnist_utils); + TrainNet(nn, dl); + constexpr DataType kExpectedAccuracy = 98; + DataType accuracy = mnist_utils.ComputeAccuracy(nn); + EXPECT_GE(accuracy, kExpectedAccuracy) << accuracy << std::endl; +} + +int main(int argc, char** argv) { + try { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); + } catch (std::exception& e) { + std::cerr << e.what() << std::endl; + } catch (...) {} +} \ No newline at end of file