diff --git a/configuration.json b/configuration.json deleted file mode 100644 index dbcfc60..0000000 --- a/configuration.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "server": { - "path": "site", - "port": 9555 - }, - "storage": { - "database": "log_analysis", - "username": "postgres", - "password": "postgres", - "host": "localhost", - "port": 5432 - } -} diff --git a/configuration.yaml b/configuration.yaml index 083a04e..7d00573 100644 --- a/configuration.yaml +++ b/configuration.yaml @@ -29,6 +29,14 @@ components_manager: defaults: HTTP_CLIENT_CONNECTION_POOL_SIZE: 1000 + # Storage + storage-wrapper: + database: $storage-database + host: $storage-host + port: $storage-port + username: $storage-username + password: $storage-password + # Static content fs-cache-main: dir: /var/www diff --git a/configuration_vars.yaml b/configuration_vars.yaml index fc36abf..5dfca6a 100644 --- a/configuration_vars.yaml +++ b/configuration_vars.yaml @@ -6,4 +6,12 @@ is_testing: false server-port: 8080 +# Storage +storage-database: log_analysis +storage-host: localhost +storage-port: 5432 +storage-username: postgres +storage-password: postgres + +# Static content site-directory: site diff --git a/log_analysis/CMakeLists.txt b/log_analysis/CMakeLists.txt index 81dbad1..6e4846a 100644 --- a/log_analysis/CMakeLists.txt +++ b/log_analysis/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(controllers) +add_subdirectory(data_models) add_subdirectory(json) add_subdirectory(server) add_subdirectory(storage) diff --git a/log_analysis/controllers/CMakeLists.txt b/log_analysis/controllers/CMakeLists.txt index c6a01b6..6324894 100644 --- a/log_analysis/controllers/CMakeLists.txt +++ b/log_analysis/controllers/CMakeLists.txt @@ -11,7 +11,8 @@ add_library(${PROJECT_NAME} ${SOURCES}) target_include_directories(${PROJECT_NAME} PUBLIC include) # Подключаем зависимости -target_link_libraries(${PROJECT_NAME} +target_link_libraries(${PROJECT_NAME} + la_data_models la_json la_storage la_utils diff --git a/log_analysis/controllers/src/anomaly_logs/details/fields.h b/log_analysis/controllers/src/anomaly_logs/details/fields.h index 3ada610..4328f2e 100644 --- a/log_analysis/controllers/src/anomaly_logs/details/fields.h +++ b/log_analysis/controllers/src/anomaly_logs/details/fields.h @@ -11,7 +11,7 @@ namespace controllers::anomaly_logs::fields { /// @defgroup Поля ответа /// @{ constexpr std::string_view id = "id"; - constexpr std::string_view class_id = "class_id"; + constexpr std::string_view cluster_id = "cluster_id"; constexpr std::string_view data = "data"; /// @} } diff --git a/log_analysis/controllers/src/anomaly_logs/details/query.cpp b/log_analysis/controllers/src/anomaly_logs/details/query.cpp deleted file mode 100644 index bdbf3ff..0000000 --- a/log_analysis/controllers/src/anomaly_logs/details/query.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "query.h" - -#include - -#include - -namespace controllers::anomaly_logs::details { - std::vector GetAnomalyLogs(int64_t last_id) { - auto transaction = storage::Transaction::Create(); - - std::ostringstream query; - query << "select id, cluster_id, data from anomaly_logs where id > " << last_id << ";"; - - auto pqxx_result = transaction.Do(query.str()); - - std::vector result; - result.reserve(pqxx_result.size()); - - for (const auto& row : pqxx_result) { - AnomalyLog anomaly_log{ - row[0].as(), - row[1].as(), - row[2].as(), - }; - - result.push_back(std::move(anomaly_log)); - } - - return result; - } -} diff --git a/log_analysis/controllers/src/anomaly_logs/details/query.h b/log_analysis/controllers/src/anomaly_logs/details/query.h deleted file mode 100644 index 8405382..0000000 --- a/log_analysis/controllers/src/anomaly_logs/details/query.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include - -namespace controllers::anomaly_logs::details { - struct AnomalyLog { - - }; - - std::vector GetAnomalyLogs(int64_t last_id); -} diff --git a/log_analysis/controllers/src/anomaly_logs/get.cpp b/log_analysis/controllers/src/anomaly_logs/get.cpp index 16b3bdd..e9155c0 100644 --- a/log_analysis/controllers/src/anomaly_logs/get.cpp +++ b/log_analysis/controllers/src/anomaly_logs/get.cpp @@ -1,23 +1,31 @@ #include +#include +#include #include #include "details/fields.h" -#include "details/query.h" using namespace controllers::anomaly_logs; json Get::operator()(const json& parameters) const { + // Проверяем наличие необходимых полей utils::CheckForRequiredFields(parameters, { fields::last_id }); - auto last_index = parameters[fields::last_id].get(); + // Создаём транзакцию + auto transaction = storage::Transaction::Create(); - auto anomaly_logs = details::GetAnomalyLogs(last_index); + // Создаём модель + data_models::AnomalyLogsModel anomaly_logs_model(transaction); + // Запрашиваем логи + auto anomaly_logs = anomaly_logs_model.GetAll(parameters[fields::last_id]); + + // Формируем ответ json result = json::array(); for (auto& log : anomaly_logs) { json item = json::object(); item[fields::id] = log.id; - item[fields::class_id] = log.class_id; + item[fields::cluster_id] = log.cluster_id; item[fields::data] = std::move(log.data); result.push_back(std::move(item)); diff --git a/log_analysis/controllers/src/logs_clusters/create.cpp b/log_analysis/controllers/src/logs_clusters/create.cpp index 57f6dd7..5db0ff1 100644 --- a/log_analysis/controllers/src/logs_clusters/create.cpp +++ b/log_analysis/controllers/src/logs_clusters/create.cpp @@ -1,15 +1,35 @@ #include +#include +#include #include #include "details/fields.h" -#include "details/query.h" using namespace controllers::logs_clusters; json Create::operator()(const json& parameters) const { + // Проверяем наличие необходимых полей utils::CheckForRequiredFields(parameters, { fields::id, fields::name, fields::description, fields::recommendation }); - details::CreateLogsCluster({parameters[fields::id], parameters[fields::name], parameters[fields::description], parameters[fields::recommendation]}); + // Создаём транзакцию + auto transaction = storage::Transaction::Create(); + + // Создаём модель + data_models::LogsClustersModel logs_clusters_model(transaction); + + // Проверяем, что кластер с указанным id не существует + int64_t id = parameters[fields::id]; + if (logs_clusters_model.Has(id)) { + throw std::runtime_error("Cluster with specified id already exists"); + } + + // Создаём кластер + logs_clusters_model.Create({ + id, + parameters[fields::name], + parameters[fields::description], + parameters[fields::recommendation] + }); return {}; } diff --git a/log_analysis/controllers/src/logs_clusters/details/query.cpp b/log_analysis/controllers/src/logs_clusters/details/query.cpp deleted file mode 100644 index 1dc3f52..0000000 --- a/log_analysis/controllers/src/logs_clusters/details/query.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "query.h" - -#include - -#include - -namespace controllers::logs_clusters::details { - std::vector GetLogsClusters() { - auto transaction = storage::Transaction::Create(); - - std::ostringstream query; - query << "select id, name, description, recommendation from logs_clusters;"; - - auto pqxx_result = transaction.Do(query.str()); - - std::vector result; - result.reserve(pqxx_result.size()); - - for (const auto& row : pqxx_result) { - LogsCluster logs_cluster{ - row[0].as(), - row[1].as(), - row[2].as(), - row[3].as(), - }; - - result.push_back(std::move(logs_cluster)); - } - - return result; - } - - void CreateLogsCluster(const LogsCluster& cluster) { - std::ostringstream query; - query << "insert into logs_clusters (id, name, description, recommendation) values (" - << cluster.id << ", " - << "'" << cluster.name << "', " - << "'" << cluster.description << "', " - << "'" << cluster.recommendation << "'" - << ");"; - - PostgreSqlStorage::Get().Do(query.str()); - } -} diff --git a/log_analysis/controllers/src/logs_clusters/details/query.h b/log_analysis/controllers/src/logs_clusters/details/query.h deleted file mode 100644 index 1363714..0000000 --- a/log_analysis/controllers/src/logs_clusters/details/query.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -namespace controllers::logs_clusters::details { - struct LogsCluster { - int64_t id; - std::string name; - std::string description; - std::string recommendation; - }; - - std::vector GetLogsClusters(); - - void CreateLogsCluster(const LogsCluster& cluster); -} diff --git a/log_analysis/controllers/src/logs_clusters/get.cpp b/log_analysis/controllers/src/logs_clusters/get.cpp index 9540965..d7579e5 100644 --- a/log_analysis/controllers/src/logs_clusters/get.cpp +++ b/log_analysis/controllers/src/logs_clusters/get.cpp @@ -1,14 +1,22 @@ #include -#include +#include +#include #include "details/fields.h" -#include "details/query.h" using namespace controllers::logs_clusters; json Get::operator()(const json& parameters) const { - auto clusters = details::GetLogsClusters(); + // Создаём транзакцию + auto transaction = storage::Transaction::Create(); + // Создаём модель + data_models::LogsClustersModel logs_clusters_model(transaction); + + // Запрашиваем кластеры + auto clusters = logs_clusters_model.GetAll(); + + // Формируем ответ json result = json::array(); for (auto& cluster : clusters) { json item = json::object(); diff --git a/log_analysis/controllers/src/statistics/details/query.cpp b/log_analysis/controllers/src/statistics/details/query.cpp deleted file mode 100644 index 09bed80..0000000 --- a/log_analysis/controllers/src/statistics/details/query.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "query.h" - -#include - -#include - -namespace controllers::statistics::details { - std::vector GetStatistics() { - auto transaction = storage::Transaction::Create(); - - std::ostringstream query; - query << "select name, value from statistics;"; - - auto pqxx_result = transaction.Do(query.str()); - - std::vector result; - result.reserve(pqxx_result.size()); - - for (const auto& row : pqxx_result) { - Statistic statistic{ row[0].as(), row[1].as() }; - result.push_back(std::move(statistic)); - } - - return result; - } -} diff --git a/log_analysis/controllers/src/statistics/details/query.h b/log_analysis/controllers/src/statistics/details/query.h deleted file mode 100644 index 6247420..0000000 --- a/log_analysis/controllers/src/statistics/details/query.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include -#include - -namespace controllers::statistics::details { - struct Statistic { - std::string name; - size_t value; - }; - - std::vector GetStatistics(); -} diff --git a/log_analysis/controllers/src/statistics/get.cpp b/log_analysis/controllers/src/statistics/get.cpp index e7872a6..edbe5da 100644 --- a/log_analysis/controllers/src/statistics/get.cpp +++ b/log_analysis/controllers/src/statistics/get.cpp @@ -1,12 +1,20 @@ #include - -#include "details/query.h" +#include +#include using namespace controllers::statistics; json Get::operator()(const json& parameters) const { - auto statistics = details::GetStatistics(); + // Создаём транзакцию + auto transaction = storage::Transaction::Create(); + // Создаём модель + data_models::StatisticsModel statistics_model(transaction); + + // Запрашиваем статистику + auto statistics = statistics_model.GetAll(); + + // Формируем ответ json result; for (auto& statistic : statistics) { result[std::move(statistic.name)] = statistic.value; diff --git a/log_analysis/data_models/CMakeLists.txt b/log_analysis/data_models/CMakeLists.txt new file mode 100644 index 0000000..620956b --- /dev/null +++ b/log_analysis/data_models/CMakeLists.txt @@ -0,0 +1,18 @@ +# Модуль моделей данных +project(la_data_models) + +# Собираем исходные файлы +file(GLOB_RECURSE SOURCES src/*.cpp) + +# Создаём библиотеку +add_library(${PROJECT_NAME} ${SOURCES}) + +# Указываем директории с заголовочными файлами +target_include_directories(${PROJECT_NAME} PUBLIC include) + +# Подключаем зависимости +target_link_libraries(${PROJECT_NAME} + la_json + la_storage + la_utils +) diff --git a/log_analysis/data_models/include/data_models/anomaly_logs.h b/log_analysis/data_models/include/data_models/anomaly_logs.h new file mode 100644 index 0000000..7aaa17e --- /dev/null +++ b/log_analysis/data_models/include/data_models/anomaly_logs.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +#include + +namespace data_models { + /// @brief Аномальный лог + struct AnomalyLog { + int64_t id; ///< Идентификатор + int64_t cluster_id; ///< Идентификатор кластера + std::string data; ///< Содержимое лога + }; + + /// @brief Модель аномальных логов + class AnomalyLogsModel { + public: + /// @brief Конструктор + /// @param[in] transaction Транзакция + AnomalyLogsModel(storage::Transaction& transaction); + + /// @brief Проверка существования записи по идентификатору + /// @param[in] id Идентификатор + /// @returns Существование записи + bool Has(int64_t id) const; + + /// @brief Получение всех записей с идентификатором больше указанного + /// @details Если надо получить все записи, необходимо передать идентификатор -1 + /// @param[in] last_id Идентификатор + /// @returns Записи + std::vector GetAll(int64_t last_id) const; + + /// @brief Получение записей с указанным идентификатором кластера + /// @param[in] cluster_id Идентификатор кластера + /// @returns Записи + std::vector GetByClusterId(int64_t cluster_id) const; + + private: + storage::Transaction& transaction_; ///< Транзакция + }; +} diff --git a/log_analysis/data_models/include/data_models/logs_clusters.h b/log_analysis/data_models/include/data_models/logs_clusters.h new file mode 100644 index 0000000..0144835 --- /dev/null +++ b/log_analysis/data_models/include/data_models/logs_clusters.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#include + +namespace data_models { + /// @brief Кластер логов + struct LogsCluster { + int64_t id; ///< Идентификатор + std::string name; ///< Название + std::string description; ///< Описание + std::string recommendation; ///< Рекомендация + }; + + /// @brief Модель кластеров логов + class LogsClustersModel { + public: + /// @brief Конструктор + /// @param[in] transaction Транзакция + LogsClustersModel(storage::Transaction& transaction); + + /// @brief Проверка существования записи по идентификатору + /// @param[in] id Идентификатор + /// @returns Существование записи + bool Has(int64_t id) const; + + /// @brief Получение всех записей + /// @returns Записи + std::vector GetAll() const; + + /// @brief Создание кластера + /// @param[in] cluster Кластер + void Create(const LogsCluster& cluster); + private: + storage::Transaction& transaction_; ///< Транзакция + }; +} diff --git a/log_analysis/data_models/include/data_models/statistics.h b/log_analysis/data_models/include/data_models/statistics.h new file mode 100644 index 0000000..cff6d4c --- /dev/null +++ b/log_analysis/data_models/include/data_models/statistics.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include + +namespace data_models { + /// @brief Статистика + struct Statistic { + std::string name; ///< Название + size_t value; ///< Значение + }; + + /// @brief Модель статистик + class StatisticsModel { + public: + /// @brief Конструктор + /// @param[in] transaction Транзакция + StatisticsModel(storage::Transaction& transaction); + + /// @brief Получение всех записей + /// @returns Записи + std::vector GetAll() const; + private: + storage::Transaction& transaction_; ///< Транзакция + }; +} diff --git a/log_analysis/data_models/src/anomaly_logs.cpp b/log_analysis/data_models/src/anomaly_logs.cpp new file mode 100644 index 0000000..7c0df80 --- /dev/null +++ b/log_analysis/data_models/src/anomaly_logs.cpp @@ -0,0 +1,51 @@ +#include + +#include + +using namespace data_models; + +namespace { + std::vector ConvertAnomalyLogs(pqxx::result pqxx_result) { + std::vector result; + result.reserve(pqxx_result.size()); + + for (const auto& row : pqxx_result) { + AnomalyLog anomaly_log{ + row[0].as(), + row[1].as(), + row[2].as(), + }; + + result.push_back(std::move(anomaly_log)); + } + + return result; + } +} + +AnomalyLogsModel::AnomalyLogsModel(storage::Transaction& transaction) + : transaction_(transaction) {} + +bool AnomalyLogsModel::Has(int64_t id) const { + std::ostringstream query; + query << "select exists(*) from anomaly_logs where id = " << id << ";"; + + auto pqxx_result = transaction_.Do(query.str()); + return pqxx_result[0][0].as(); +} + +std::vector AnomalyLogsModel::GetAll(int64_t last_id) const { + std::ostringstream query; + query << "select id, cluster_id, data from anomaly_logs where id > " << last_id << ";"; + + auto pqxx_result = transaction_.Do(query.str()); + return ConvertAnomalyLogs(std::move(pqxx_result)); +} + +std::vector AnomalyLogsModel::GetByClusterId(int64_t cluster_id) const { + std::ostringstream query; + query << "select id, cluster_id, data from anomaly_logs where cluster_id = " << cluster_id << ";"; + + auto pqxx_result = transaction_.Do(query.str()); + return ConvertAnomalyLogs(std::move(pqxx_result)); +} diff --git a/log_analysis/data_models/src/logs_clusters.cpp b/log_analysis/data_models/src/logs_clusters.cpp new file mode 100644 index 0000000..6ad6ca2 --- /dev/null +++ b/log_analysis/data_models/src/logs_clusters.cpp @@ -0,0 +1,51 @@ +#include + +#include + +using namespace data_models; + +LogsClustersModel::LogsClustersModel(storage::Transaction& transaction) + : transaction_(transaction) {} + +bool LogsClustersModel::Has(int64_t id) const { + std::ostringstream query; + query << "select id from logs_clusters where id = " << id << ";"; + + auto pqxx_result = transaction_.Do(query.str()); + return !pqxx_result.empty(); +} + +std::vector LogsClustersModel::GetAll() const { + std::ostringstream query; + query << "select id, name, description, recommendation from logs_clusters;"; + + auto pqxx_result = transaction_.Do(query.str()); + + std::vector result; + result.reserve(pqxx_result.size()); + + for (const auto& row : pqxx_result) { + LogsCluster logs_cluster{ + row[0].as(), + row[1].as(), + row[2].as(), + row[3].as(), + }; + + result.push_back(std::move(logs_cluster)); + } + + return result; +} + +void LogsClustersModel::Create(const LogsCluster& cluster) { + std::ostringstream query; + query << "insert into logs_clusters (id, name, description, recommendation) values " + << "(" << cluster.id << ", " + << "'" << cluster.name << "', " + << "'" << cluster.description << "', " + << "'" << cluster.recommendation << "'" + << ");"; + + transaction_.Do(query.str()); +} diff --git a/log_analysis/data_models/src/statistics.cpp b/log_analysis/data_models/src/statistics.cpp new file mode 100644 index 0000000..e0fd01c --- /dev/null +++ b/log_analysis/data_models/src/statistics.cpp @@ -0,0 +1,29 @@ +#include + +#include + +using namespace data_models; + +StatisticsModel::StatisticsModel(storage::Transaction& transaction) + : transaction_(transaction) {} + +std::vector StatisticsModel::GetAll() const { + std::ostringstream query; + query << "select name, value from statistics;"; + + auto pqxx_result = transaction_.Do(query.str()); + + std::vector result; + result.reserve(pqxx_result.size()); + + for (const auto& row : pqxx_result) { + Statistic statistic{ + row[0].as(), + row[1].as() + }; + + result.push_back(std::move(statistic)); + } + + return result; +} diff --git a/log_analysis/server/CMakeLists.txt b/log_analysis/server/CMakeLists.txt index 58b4efb..3fafe0f 100644 --- a/log_analysis/server/CMakeLists.txt +++ b/log_analysis/server/CMakeLists.txt @@ -12,9 +12,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/i # Подключаем зависимости target_link_libraries(${PROJECT_NAME} - la_configuration la_controllers - la_json - la_storage userver-core ) diff --git a/log_analysis/server/src/details/storage_wrapper.cpp b/log_analysis/server/src/details/storage_wrapper.cpp index 6d86a79..c31dcf7 100644 --- a/log_analysis/server/src/details/storage_wrapper.cpp +++ b/log_analysis/server/src/details/storage_wrapper.cpp @@ -1,5 +1,7 @@ #include "storage_wrapper.h" +#include + #include using namespace server::details; @@ -7,14 +9,36 @@ using namespace server::details; StorageWrapper::StorageWrapper(const userver::components::ComponentConfig& config, const userver::components::ComponentContext& context) : userver::components::LoggableComponentBase(config, context) { - auto settings = config[kName]; - storage::ConnectionData connection_data; - connection_data.database = settings["database"].As(); - connection_data.username = settings["username"].As(); - connection_data.password = settings["password"].As(); - connection_data.host = settings["host"].As(); - connection_data.port = settings["port"].As(); + connection_data.database = config["database"].As(); + connection_data.username = config["username"].As(); + connection_data.password = config["password"].As(); + connection_data.host = config["host"].As(); + connection_data.port = config["port"].As(); storage::Storage::Connect(connection_data); } + +userver::yaml_config::Schema StorageWrapper::GetStaticConfigSchema() { + return userver::yaml_config::MergeSchemas(R"( + type: object + description: storage wrapper component + additionalProperties: false + properties: + database: + type: string + description: database + host: + type: string + description: host + port: + type: string + description: port + username: + type: string + description: username + password: + type: string + description: password + )"); +} diff --git a/log_analysis/server/src/details/storage_wrapper.h b/log_analysis/server/src/details/storage_wrapper.h index 8fa3473..d5fda2a 100644 --- a/log_analysis/server/src/details/storage_wrapper.h +++ b/log_analysis/server/src/details/storage_wrapper.h @@ -2,6 +2,7 @@ #include #include +#include namespace server::details { /// @brief Обёртка-компонент над хранилищем @@ -14,5 +15,9 @@ namespace server::details { /// @param[in] config Конфигурация /// @param[in] context Контекст StorageWrapper(const userver::components::ComponentConfig& config, const userver::components::ComponentContext& context); + + /// @brief Получение схемы статической конфигурации + /// @returns Схема статической конфигурации + static userver::yaml_config::Schema GetStaticConfigSchema(); }; } diff --git a/log_analysis/utils/src/logger.cpp b/log_analysis/utils/src/logger.cpp index 64c156f..7c2ba08 100644 --- a/log_analysis/utils/src/logger.cpp +++ b/log_analysis/utils/src/logger.cpp @@ -1,4 +1,4 @@ -#include > +#include Logger& Logger::Get() { static Logger logger;