From c0c3197cf2183a267c5e7c3f401e8778b20d136d Mon Sep 17 00:00:00 2001 From: Kirill Volokh Date: Fri, 23 Feb 2024 20:05:05 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D1=81=D0=BB=D0=BE=D1=8F=20=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log_analysis/CMakeLists.txt | 1 - log_analysis/configuration/CMakeLists.txt | 16 ---------- .../include/configuration/configuration.h | 11 ------- log_analysis/configuration/notes.md | 16 ---------- .../configuration/src/configuration.cpp | 32 ------------------- .../src/anomaly_logs/details/query.cpp | 6 ++-- .../src/anomaly_logs/details/query.h | 4 +-- .../src/logs_clusters/details/query.cpp | 6 ++-- .../src/statistics/details/query.cpp | 6 ++-- .../server/src/details/storage_wrapper.cpp | 20 ++++++++++++ .../server/src/details/storage_wrapper.h | 18 +++++++++++ log_analysis/server/src/main.cpp | 30 ++--------------- .../storage/postgresql/postgresql_storage.h | 19 ----------- .../storage/include/storage/storage.h | 27 ++++++++++++++++ .../storage/include/storage/transaction.h | 25 +++++++++++++++ .../storage/src/postgresql_storage.cpp | 32 ------------------- log_analysis/storage/src/storage.cpp | 21 ++++++++++++ log_analysis/storage/src/transaction.cpp | 23 +++++++++++++ 18 files changed, 150 insertions(+), 163 deletions(-) delete mode 100644 log_analysis/configuration/CMakeLists.txt delete mode 100644 log_analysis/configuration/include/configuration/configuration.h delete mode 100644 log_analysis/configuration/notes.md delete mode 100644 log_analysis/configuration/src/configuration.cpp create mode 100644 log_analysis/server/src/details/storage_wrapper.cpp create mode 100644 log_analysis/server/src/details/storage_wrapper.h delete mode 100644 log_analysis/storage/include/storage/postgresql/postgresql_storage.h create mode 100644 log_analysis/storage/include/storage/storage.h create mode 100644 log_analysis/storage/include/storage/transaction.h delete mode 100644 log_analysis/storage/src/postgresql_storage.cpp create mode 100644 log_analysis/storage/src/storage.cpp create mode 100644 log_analysis/storage/src/transaction.cpp diff --git a/log_analysis/CMakeLists.txt b/log_analysis/CMakeLists.txt index 071a36b..81dbad1 100644 --- a/log_analysis/CMakeLists.txt +++ b/log_analysis/CMakeLists.txt @@ -1,4 +1,3 @@ -add_subdirectory(configuration) add_subdirectory(controllers) add_subdirectory(json) add_subdirectory(server) diff --git a/log_analysis/configuration/CMakeLists.txt b/log_analysis/configuration/CMakeLists.txt deleted file mode 100644 index f7a6347..0000000 --- a/log_analysis/configuration/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Модуль для работы с конфигурационными файлами -project(la_configuration) - -# Собираем исходные файлы -file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) - -# Создаём библиотеку -add_library(${PROJECT_NAME} ${SOURCES}) - -# Указываем директории с заголовочными файлами -target_include_directories(${PROJECT_NAME} PUBLIC include) - -# Подключаем зависимости -target_link_libraries(${PROJECT_NAME} - la_json -) diff --git a/log_analysis/configuration/include/configuration/configuration.h b/log_analysis/configuration/include/configuration/configuration.h deleted file mode 100644 index 3db0ea2..0000000 --- a/log_analysis/configuration/include/configuration/configuration.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include - -#include - -/// @brief Парсинг конфигурационного файла -/// @param[in] path_to_configuration Путь к конфигурационному файлу -/// @param[in] required_keys Список обязательных ключей в формате { "key1": ["subkey1", ...], "key2": ... } -/// @returns Конфигурация в формате json -json ParseConfiguration(const std::string& path_to_configuration, const json& required_keys); diff --git a/log_analysis/configuration/notes.md b/log_analysis/configuration/notes.md deleted file mode 100644 index 89e21a5..0000000 --- a/log_analysis/configuration/notes.md +++ /dev/null @@ -1,16 +0,0 @@ -# Модуль configuration -Конфигурационный файл должен иметь следующий формат: -```json -{ - "key1": { - "subkey1": "value1", - "subkey2": "value2", - ... - }, - "key2": { - "subkey1": "value1", - ... - }, - ... -} -``` diff --git a/log_analysis/configuration/src/configuration.cpp b/log_analysis/configuration/src/configuration.cpp deleted file mode 100644 index 5d02bd7..0000000 --- a/log_analysis/configuration/src/configuration.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include - -#include - -json ParseConfiguration(const std::string& path_to_configuration, const json& required_keys) { - std::ifstream input(path_to_configuration); - auto configuration = json::parse(input); - - for (const auto& [key, value] : required_keys.items()) { - // Проверяем наличие ключа на "нулевом" уровне - if (!configuration.contains(key)) { - std::ostringstream message; - message << "configuration doesn't contain required key '" << key << "'"; - throw std::runtime_error(message.str()); - } - - // Проверяем наличие вложенных ключей - auto configuration_value = configuration[key]; - - for (const auto& item : value) { - if (!configuration_value.contains(item)) { - std::ostringstream message; - message << "configuration doesn't contain required key '" << item << "' in '" << key << "'"; - throw std::runtime_error(message.str()); - } - } - } - - return configuration; -} diff --git a/log_analysis/controllers/src/anomaly_logs/details/query.cpp b/log_analysis/controllers/src/anomaly_logs/details/query.cpp index f4932d7..bdbf3ff 100644 --- a/log_analysis/controllers/src/anomaly_logs/details/query.cpp +++ b/log_analysis/controllers/src/anomaly_logs/details/query.cpp @@ -2,14 +2,16 @@ #include -#include +#include namespace controllers::anomaly_logs::details { std::vector GetAnomalyLogs(int64_t last_id) { + auto transaction = storage::Transaction::Create(); + std::ostringstream query; query << "select id, cluster_id, data from anomaly_logs where id > " << last_id << ";"; - auto pqxx_result = PostgreSqlStorage::Get().Do(query.str()); + auto pqxx_result = transaction.Do(query.str()); std::vector result; result.reserve(pqxx_result.size()); diff --git a/log_analysis/controllers/src/anomaly_logs/details/query.h b/log_analysis/controllers/src/anomaly_logs/details/query.h index 1f3dab1..8405382 100644 --- a/log_analysis/controllers/src/anomaly_logs/details/query.h +++ b/log_analysis/controllers/src/anomaly_logs/details/query.h @@ -5,9 +5,7 @@ namespace controllers::anomaly_logs::details { struct AnomalyLog { - int64_t id; - int64_t class_id; - std::string data; + }; std::vector GetAnomalyLogs(int64_t last_id); diff --git a/log_analysis/controllers/src/logs_clusters/details/query.cpp b/log_analysis/controllers/src/logs_clusters/details/query.cpp index 8948226..1dc3f52 100644 --- a/log_analysis/controllers/src/logs_clusters/details/query.cpp +++ b/log_analysis/controllers/src/logs_clusters/details/query.cpp @@ -2,14 +2,16 @@ #include -#include +#include namespace controllers::logs_clusters::details { std::vector GetLogsClusters() { + auto transaction = storage::Transaction::Create(); + std::ostringstream query; query << "select id, name, description, recommendation from logs_clusters;"; - auto pqxx_result = PostgreSqlStorage::Get().Do(query.str()); + auto pqxx_result = transaction.Do(query.str()); std::vector result; result.reserve(pqxx_result.size()); diff --git a/log_analysis/controllers/src/statistics/details/query.cpp b/log_analysis/controllers/src/statistics/details/query.cpp index c111801..09bed80 100644 --- a/log_analysis/controllers/src/statistics/details/query.cpp +++ b/log_analysis/controllers/src/statistics/details/query.cpp @@ -2,14 +2,16 @@ #include -#include +#include namespace controllers::statistics::details { std::vector GetStatistics() { + auto transaction = storage::Transaction::Create(); + std::ostringstream query; query << "select name, value from statistics;"; - auto pqxx_result = PostgreSqlStorage::Get().Do(query.str()); + auto pqxx_result = transaction.Do(query.str()); std::vector result; result.reserve(pqxx_result.size()); diff --git a/log_analysis/server/src/details/storage_wrapper.cpp b/log_analysis/server/src/details/storage_wrapper.cpp new file mode 100644 index 0000000..6d86a79 --- /dev/null +++ b/log_analysis/server/src/details/storage_wrapper.cpp @@ -0,0 +1,20 @@ +#include "storage_wrapper.h" + +#include + +using namespace server::details; + +StorageWrapper::StorageWrapper(const userver::components::ComponentConfig& config, + const userver::components::ComponentContext& context) + : userver::components::LoggableComponentBase(config, context) { + auto settings = config[kName]; + + storage::ConnectionData connection_data; + connection_data.database = settings["database"].As(); + connection_data.username = settings["username"].As(); + connection_data.password = settings["password"].As(); + connection_data.host = settings["host"].As(); + connection_data.port = settings["port"].As(); + + storage::Storage::Connect(connection_data); +} diff --git a/log_analysis/server/src/details/storage_wrapper.h b/log_analysis/server/src/details/storage_wrapper.h new file mode 100644 index 0000000..8fa3473 --- /dev/null +++ b/log_analysis/server/src/details/storage_wrapper.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace server::details { + /// @brief Обёртка-компонент над хранилищем + class StorageWrapper : public userver::components::LoggableComponentBase { + public: + /// @brief Название компонента + static constexpr std::string_view kName = "storage-wrapper"; + + /// @brief Конструктор + /// @param[in] config Конфигурация + /// @param[in] context Контекст + StorageWrapper(const userver::components::ComponentConfig& config, const userver::components::ComponentContext& context); + }; +} diff --git a/log_analysis/server/src/main.cpp b/log_analysis/server/src/main.cpp index 04e708f..11bb744 100644 --- a/log_analysis/server/src/main.cpp +++ b/log_analysis/server/src/main.cpp @@ -11,42 +11,16 @@ #include #include -#include #include -#include #include "details/controller_wrapper.h" +#include "details/storage_wrapper.h" #include "details/details.h" void Log(LogLevel level, const char* message) { std::cout << message << std::endl; } -void ConnectToStorage(const json& configuration) { - Logger::Get().Write(LogLevel::Info, "connecting to postgresql..."); - - const auto& parameters = configuration["storage"]; - - ConnectionData connection_data; - connection_data.database = parameters["database"].get(); - connection_data.username = parameters["username"].get(); - connection_data.password = parameters["password"].get(); - connection_data.host = parameters["host"].get(); - connection_data.port = std::to_string(parameters["port"].get()); - - PostgreSqlStorage::Connect(connection_data); - - Logger::Get().Write(LogLevel::Info, "connected to postgresql"); -} - -json ParseConfiguration(const std::string& path_to_configuration) { - json required_keys; - required_keys["server"] = { "path", "port" }; - required_keys["storage"] = { "database", "username", "password", "host", "port" }; - - return ParseConfiguration(path_to_configuration, required_keys); -} - int main(int argc, char** argv) { if (argc < 2) { std::cout << "usage: " << argv[0] << " " << std::endl; @@ -66,6 +40,8 @@ int main(int argc, char** argv) { // static content .Append("fs-cache-main") .Append() + // storage + .Append() // anomaly logs .Append>() // clusters diff --git a/log_analysis/storage/include/storage/postgresql/postgresql_storage.h b/log_analysis/storage/include/storage/postgresql/postgresql_storage.h deleted file mode 100644 index 381ad29..0000000 --- a/log_analysis/storage/include/storage/postgresql/postgresql_storage.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include - -#include - -struct ConnectionData { - std::string database, username, password, host, port; -}; - -class PostgreSqlStorage { -public: - static PostgreSqlStorage& Get(); - static void Connect(const ConnectionData& connection_data); - pqxx::result Do(std::string_view query) const; -private: - std::unique_ptr connection_; -}; diff --git a/log_analysis/storage/include/storage/storage.h b/log_analysis/storage/include/storage/storage.h new file mode 100644 index 0000000..c2e4653 --- /dev/null +++ b/log_analysis/storage/include/storage/storage.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include + +#include + +namespace storage { + /// @brief Данные подключения + struct ConnectionData { + std::string database, username, password, host, port; + }; + + class Storage { + public: + static Storage& Get(); + + static void Connect(const ConnectionData& connection_data); + + friend class Transaction; + + private: + std::unique_ptr connection_; + }; +} diff --git a/log_analysis/storage/include/storage/transaction.h b/log_analysis/storage/include/storage/transaction.h new file mode 100644 index 0000000..0ed5434 --- /dev/null +++ b/log_analysis/storage/include/storage/transaction.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace storage { + class Transaction { + public: + ~Transaction(); + + Transaction(Transaction&&) = default; + Transaction& operator=(Transaction&&) = default; + + static Transaction Create(); + + pqxx::result Do(std::string_view query) const; + + private: + Transaction() = default; + + private: + std::unique_ptr transaction_; + }; +} diff --git a/log_analysis/storage/src/postgresql_storage.cpp b/log_analysis/storage/src/postgresql_storage.cpp deleted file mode 100644 index d6fa1cf..0000000 --- a/log_analysis/storage/src/postgresql_storage.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include - -#include - -PostgreSqlStorage& PostgreSqlStorage::Get() { - static PostgreSqlStorage storage; - return storage; -} - -void PostgreSqlStorage::Connect(const ConnectionData& connection_data) { - std::ostringstream options; - options << "dbname = " << connection_data.database - << " user = " << connection_data.username - << " password = " << connection_data.password - << " host = " << connection_data.host - << " port = " << connection_data.port; - - Get().connection_ = std::make_unique(options.str()); -} - -pqxx::result PostgreSqlStorage::Do(std::string_view query) const { - // Создаём транзакцию - pqxx::work work(*connection_); - - // Выполняем запрос - pqxx::result result = work.exec(query); - - // Завершаем транзакцию - work.commit(); - - return result; -} \ No newline at end of file diff --git a/log_analysis/storage/src/storage.cpp b/log_analysis/storage/src/storage.cpp new file mode 100644 index 0000000..2eae254 --- /dev/null +++ b/log_analysis/storage/src/storage.cpp @@ -0,0 +1,21 @@ +#include + +#include + +using namespace storage; + +Storage& Storage::Get() { + static Storage storage; + return storage; +} + +void Storage::Connect(const ConnectionData& connection_data) { + std::ostringstream options; + options << "dbname = " << connection_data.database + << " user = " << connection_data.username + << " password = " << connection_data.password + << " host = " << connection_data.host + << " port = " << connection_data.port; + + Get().connection_ = std::make_unique(options.str()); +} diff --git a/log_analysis/storage/src/transaction.cpp b/log_analysis/storage/src/transaction.cpp new file mode 100644 index 0000000..b6c8d2b --- /dev/null +++ b/log_analysis/storage/src/transaction.cpp @@ -0,0 +1,23 @@ +#include + +#include +#include + +using namespace storage; + +Transaction::~Transaction() { + transaction_->commit(); +} + +Transaction Transaction::Create() { + auto& storage = storage::Storage::Get(); + + Transaction transaction; + transaction.transaction_ = std::make_unique(*storage.connection_); + + return transaction; +} + +pqxx::result Transaction::Do(std::string_view query) const { + return transaction_->exec(query); +} \ No newline at end of file