Добавил скрипты сборки, SQL-файлы и конфигурации

This commit is contained in:
2024-02-22 20:40:18 +03:00
parent dcbf32b1a4
commit 555a7a50d8
19 changed files with 1460 additions and 0 deletions
+22
View File
@@ -0,0 +1,22 @@
cmake_minimum_required(VERSION 3.16)
cmake_policy(SET CMP0077 NEW)
set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
# Проект Log Analysis. Названия CMake-таргетов имеют префикс "la_" (первые буквы слов в названии проекта)
project(traffic_analysis)
# Указываем переменные
set(CMAKE_CXX_STANDARD 20)
set(PROJECT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(THIRD_PARTY_DIR ${PROJECT_ROOT_DIR}/third_party)
# Подготавливаем зависимости
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/PrepareDependencies.cmake)
# Добавляем модули
add_subdirectory(configuration)
add_subdirectory(controllers)
add_subdirectory(json)
add_subdirectory(server)
add_subdirectory(storage)
add_subdirectory(utils)
+1161
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -0,0 +1,8 @@
# Подготавливаем зависимости
include(FetchContent)
include(${CMAKE_CURRENT_LIST_DIR}/PrepareNlohmannJson.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/PreparePostgreSql.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/PreparePqxx.cmake)
include(${CMAKE_CURRENT_LIST_DIR}/PrepareUserver.cmake)
+8
View File
@@ -0,0 +1,8 @@
# drogon
FetchContent_Declare(drogon
GIT_REPOSITORY https://github.com/drogonframework/drogon.git
GIT_TAG v1.9.3
)
FetchContent_MakeAvailable(drogon)
+9
View File
@@ -0,0 +1,9 @@
# nlohmann/json
FetchContent_Declare(Jsoncpp
GIT_REPOSITORY https://github.com/open-source-parsers/jsoncpp.git
GIT_TAG 1.9.5
)
FetchContent_MakeAvailable(Jsoncpp)
+11
View File
@@ -0,0 +1,11 @@
# nlohmann/json
# Отключаем сборку тестов
set(JSON_BuildTests OFF)
FetchContent_Declare(nlohmann-json
GIT_REPOSITORY https://github.com/nlohmann/json.git
GIT_TAG v3.11.2
)
FetchContent_MakeAvailable(nlohmann-json)
+10
View File
@@ -0,0 +1,10 @@
# libpq
if (WIN32)
# Для Windows необходимо скопировать библиотеки и заголовочные файлы PostgreSQL в папку third_party/postgresql
# Дополнительно необходимо вручную положить libpq.dll, libcrypto-3-x64.dll, libssl-3-x64.dll рядом с исполняемым приложением
set(PostgreSQL_INCLUDE_DIR ${THIRD_PARTY_DIR}/postgresql/include)
set(PostgreSQL_TYPE_INCLUDE_DIR ${THIRD_PARTY_DIR}/postgresql/include/server)
set(PostgreSQL_LIBRARY_DEBUG ${THIRD_PARTY_DIR}/postgresql/debug/lib/libpq.lib)
set(PostgreSQL_LIBRARY_RELEASE ${THIRD_PARTY_DIR}/postgresql/lib/libpq.lib)
endif()
+11
View File
@@ -0,0 +1,11 @@
# libpqxx
# Отключаем сборку тестов
set(SKIP_BUILD_TEST ON)
FetchContent_Declare(libpqxx
GIT_REPOSITORY https://github.com/jtv/libpqxx.git
GIT_TAG 7.8.1
)
FetchContent_MakeAvailable(libpqxx)
+13
View File
@@ -0,0 +1,13 @@
# seasocks
# Отключаем сборку тестов, демо-приложения и поддержку deflate
set(UNITTESTS OFF)
set(SEASOCKS_EXAMPLE_APP OFF)
set(DEFLATE_SUPPORT OFF)
FetchContent_Declare(seasocks
GIT_REPOSITORY https://github.com/mattgodbolt/seasocks.git
GIT_TAG v1.4.5
)
FetchContent_MakeAvailable(seasocks)
+16
View File
@@ -0,0 +1,16 @@
# userver
# Отключаем сборку тестов
set(USERVER_FEATURE_JEMALLOC OFF)
set(USERVER_FEATURE_MONGODB OFF)
set(USERVER_FEATURE_POSTGRESQL OFF)
set(USERVER_FEATURE_PATCH_LIBPQ OFF)
set(USERVER_FEATURE_REDIS OFF)
set(USERVER_FEATURE_GRPC OFF)
set(USERVER_FEATURE_CLICKHOUSE OFF)
set(USERVER_FEATURE_RABBITMQ OFF)
set(USERVER_FEATURE_UTEST OFF)
set(USERVER_FEATURE_TESTSUITE OFF)
add_subdirectory(${THIRD_PARTY_DIR}/userver)
userver_setup_environment()
+13
View File
@@ -0,0 +1,13 @@
{
"server": {
"path": "site",
"port": 9555
},
"storage": {
"database": "log_analysis",
"username": "postgres",
"password": "postgres",
"host": "localhost",
"port": 5432
}
}
+63
View File
@@ -0,0 +1,63 @@
components_manager:
task_processors: # Task processor is an executor for coroutine tasks
main-task-processor: # Make a task processor for CPU-bound coroutine tasks.
worker_threads: $worker-threads # Process tasks in 4 threads.
fs-task-processor: # Make a separate task processor for filesystem bound tasks.
worker_threads: $worker-fs-threads
default_task_processor: main-task-processor
components: # Configuring components that were registered via component_list
server:
listener: # configuring the main listening socket...
port: $server-port # ...to listen on this port and...
task_processor: main-task-processor # ...process incoming requests on this task processor.
logging:
fs-task-processor: fs-task-processor
loggers:
default:
file_path: '@stderr'
level: $logger-level
overflow_behavior: discard # Drop logs if the system is too busy to write them down.
# Dynamic config options. Cache is disabled, updates are disabled.
dynamic-config:
# For most of userver dynamic configs, defaults are used, some are overridden here.
# See userver "dynamic config" docs for what configs exist.
defaults:
HTTP_CLIENT_CONNECTION_POOL_SIZE: 1000
# Static content
fs-cache-main:
dir: /var/www
handler-static:
path: /*
method: GET
fs-cache-component: fs-cache-main
task_processor: main-task-processor
# Anomaly Logs
anomaly-logs-get:
path: /anomaly_logs
method: GET
task_processor: main-task-processor
# Logs Clusters
logs-clusters-create:
path: /logs_clusters
method: POST
task_processor: main-task-processor
logs-clusters-get:
path: /logs_clusters
method: GET
task_processor: main-task-processor
# Statistics
statistics-get:
path: /statistics
method: GET
task_processor: main-task-processor
+16
View File
@@ -0,0 +1,16 @@
# Модуль для работы с конфигурационными файлами
project(la_configuration)
# Собираем исходные файлы
file(GLOB_RECURSE SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
# Создаём библиотеку
add_library(${PROJECT_NAME} ${SOURCES})
# Указываем директории с заголовочными файлами
target_include_directories(${PROJECT_NAME} PUBLIC include)
# Подключаем зависимости
target_link_libraries(${PROJECT_NAME}
la_json
)
@@ -0,0 +1,11 @@
#pragma once
#include <string>
#include <json/json.h>
/// @brief Парсинг конфигурационного файла
/// @param[in] path_to_configuration Путь к конфигурационному файлу
/// @param[in] required_keys Список обязательных ключей в формате { "key1": ["subkey1", ...], "key2": ... }
/// @returns Конфигурация в формате json
json ParseConfiguration(const std::string& path_to_configuration, const json& required_keys);
+16
View File
@@ -0,0 +1,16 @@
# Модуль configuration
Конфигурационный файл должен иметь следующий формат:
```json
{
"key1": {
"subkey1": "value1",
"subkey2": "value2",
...
},
"key2": {
"subkey1": "value1",
...
},
...
}
```
+32
View File
@@ -0,0 +1,32 @@
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <configuration/configuration.h>
json ParseConfiguration(const std::string& path_to_configuration, const json& required_keys) {
std::ifstream input(path_to_configuration);
auto configuration = json::parse(input);
for (const auto& [key, value] : required_keys.items()) {
// Проверяем наличие ключа на "нулевом" уровне
if (!configuration.contains(key)) {
std::ostringstream message;
message << "configuration doesn't contain required key '" << key << "'";
throw std::runtime_error(message.str());
}
// Проверяем наличие вложенных ключей
auto configuration_value = configuration[key];
for (const auto& item : value) {
if (!configuration_value.contains(item)) {
std::ostringstream message;
message << "configuration doesn't contain required key '" << item << "' in '" << key << "'";
throw std::runtime_error(message.str());
}
}
}
return configuration;
}
+9
View File
@@ -0,0 +1,9 @@
worker-threads: 4
worker-fs-threads: 2
logger-level: debug
is_testing: false
server-port: 8080
site-directory: site
+5
View File
@@ -0,0 +1,5 @@
update statistics set value = 5 where name = 'anomaly';
update statistics set value = 21 where name = 'not anomaly';
update statistics set value = 26 where name = 'total';
insert into anomaly_logs (cluster_id, data) values (0, 'filesystem error'), (0, 'network error');
+26
View File
@@ -0,0 +1,26 @@
create database log_analysis;
create table statistics(
name varchar(64),
value int,
primary key (name)
);
insert into statistics (name, value) values ('total', 0), ('anomaly', 0), ('not anomaly', 0), ('undefined', 0);
create table logs_clusters(
id int,
name text,
description text,
recommendation text,
primary key (id)
);
insert into logs_clusters (id, name, description, recommendation) values (0, 'unknown', '', '');
create table anomaly_logs(
id serial,
cluster_id int references logs_clusters(id),
data text,
primary key (id)
);