Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
-193
View File
@@ -1,193 +0,0 @@
import json
from modules.transports.client import Client
class Settings:
def __init__(self):
self.host = None
self.port = None
class CommandException(Exception):
def __init__(self, message):
super().__init__(message)
class Command:
def __init__(self, name):
"""
Конструктор
:param name: Название команды
"""
self.name = name
self.settings = None
def do(self, parameters):
"""
Вызов команды
:param parameters: Параметры
:return: Результат работы
"""
raise Exception("calling unimplemented method")
def doRequest(self, request: dict):
"""
Выполнение запроса
:param request: Тело запроса
:return: Ответ
"""
# Создаём клиентский сокет и подключаемся к серверу
client = Client()
client.connect(self.settings.host, self.settings.port)
# Отправляем запрос
client.send(json.dumps(request))
# Получаем ответ
response = client.receive()
# Закрываем соединение
client.disconnect()
# Парсим ответ
response = json.loads(response)
# Проверяем статус обработки запроса
status = response["status"]
if status != "ok":
message = response["message"]
raise CommandException(f"status: {status}, message: {message}")
return response["result"]
class CommandManager:
def __init__(self, settings):
self.settings = settings
self.commands = []
self.command_name_to_index = {}
def addCommand(self, command: Command):
command.settings = self.settings
self.command_name_to_index[command.name] = len(self.commands)
self.commands.append(command)
def hasCommand(self, name) -> bool:
if str.isnumeric(name):
index = int(name)
if index < 0 or index >= len(self.commands):
return False
return True
return name in self.command_name_to_index
def getCommand(self, name):
if not self.hasCommand(name):
raise Exception("there is no command with such name")
if str.isnumeric(name):
return self.commands[int(name)]
return self.commands[self.command_name_to_index[name]]
class GetPipes(Command):
"""
Получение списка пайп
"""
def __init__(self):
super().__init__("get_pipes")
def do(self, parameters):
request = {
"handler": self.name,
"parameters": ""
}
# Ответ -- это список пайп
response = self.doRequest(request)
if len(response) == 0:
print("there are no pipes")
return
for pipe in response[:-1]:
self.printPipe(pipe)
print()
self.printPipe(response[-1])
def printPipe(self, pipe):
print(pipe["name"] + ":")
print("- description: " + pipe["description"])
print("- input data type: " + pipe["input_data_type"])
print("- output data type: " + pipe["output_data_type"])
class GetPipelines(Command):
"""
Получение списка пайплайнов
"""
def __init__(self):
super().__init__("get_pipelines")
def do(self, parameters):
request = {
"handler": self.name,
"parameters": ""
}
# Ответ -- это список пайплайнов
response = self.doRequest(request)
if len(response) == 0:
print("there are no pipelines")
return
for pipeline in response:
print(pipeline["name"] + ": " + str(pipeline["pipes"]))
class RunPipeline(Command):
"""
Запуск пайплайна
"""
def __init__(self):
super().__init__("run_pipeline")
def do(self, parameters):
pipeline = input("pipeline: ")
data = input("data: ")
request = {
"handler": self.name,
"parameters": {
"name": pipeline,
"data": data
}
}
response = self.doRequest(request)
print("result: " + str(response))
class CreatePipeline(Command):
"""
Создание пайплайна
"""
def __init__(self):
super().__init__("create_pipeline")
def do(self, parameters):
name = input("name: ")
pipes = input("pipes (separated by spaces): ")
request = {
"handler": self.name,
"parameters": {
"name": name,
"pipes": pipes.split()
}
}
self.doRequest(request)
print("created successfully")
-70
View File
@@ -1,70 +0,0 @@
import argparse
from services.console_client.commands import *
def CreateCommandManager(settings):
manager = CommandManager(settings)
manager.addCommand(GetPipes())
manager.addCommand(GetPipelines())
manager.addCommand(CreatePipeline())
manager.addCommand(RunPipeline())
return manager
def PrintHelp(command_manager):
for index, command in enumerate(command_manager.commands):
print(str(index) + ". " + command.name)
def ParseArguments():
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
# Хост сервера
parser.add_argument("--host", type=str, required=True, help="server's host")
# Порт
parser.add_argument("--port", type=int, required=True, help="server's port")
arguments = parser.parse_args()
return arguments
def CreateSettings() -> Settings:
arguments = ParseArguments()
settings = Settings()
settings.host = arguments.host
settings.port = arguments.port
return settings
def main():
settings = CreateSettings()
command_manager = CreateCommandManager(settings)
parameters = {}
PrintHelp(command_manager)
while True:
command_name = str(input("> "))
if command_name in ["q", "quit", "exit"]:
break
if command_name in ["h", "help"]:
PrintHelp(command_manager)
continue
if not command_manager.hasCommand(command_name):
print("there is no command with such name")
continue
command = command_manager.getCommand(command_name)
try:
command.do(parameters)
except Exception as e:
print(f"error: {str(e)}")
print()
if __name__ == "__main__":
main()
-19
View File
@@ -1,19 +0,0 @@
from modules.pipelines.pipe_manager import PipeManager
from modules.pipelines.pipeline_manager import PipelineManager
from modules.handlers.handler_manager import HandlerManager
from modules.storages.storage import Storage
class Context:
"""
Контекст управляющего сервиса
"""
def __init__(self):
"""
Конструктор
"""
self.pipe_manager = None
self.pipeline_manager = None
self.handler_manager = None
self.storage = None
-130
View File
@@ -1,130 +0,0 @@
import argparse
import asyncio
import json
from modules.logging.logger import *
from modules.storages.storage import Storage
from modules.transports.server import Server
from services.control.context import *
from services.control.create_context import *
from services.control.response import *
def Handle(context, request):
"""
Обработка запроса
:param context: Контекст сервиса
:param request: Запрос
:return: Ответ
"""
# Парсим запрос
try:
request = json.loads(request)
except Exception as e:
return ErrorResponse("invalid request", "failed to parse request")
# Проверяем наличие нужных ключей
for key in ["handler", "parameters"]:
if key not in request:
return ErrorResponse("invalid request", f"key '{key}' is required")
# Проверяем наличие обработчика с указанным именем
handler_name = request["handler"]
if not context.handler_manager.contains(handler_name):
return ErrorResponse("not found", f"there is no handler with name {handler_name}")
# Достаём нужного обработчика
handler = context.handler_manager.get(handler_name)
# Запускаем обработку
try:
result = handler.do(context, request["parameters"])
return SuccessResponse(result)
except Exception as e:
return ErrorResponse("handling failed", str(e))
def ParseArguments():
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
# Данные сервера
parser.add_argument("--host", type=str, required=True, help="server's host")
parser.add_argument("--port", type=int, required=True, help="server's port")
# Данные для подключения к базе данных
parser.add_argument("--psql-host", type=str, required=True, help="postgresql's host")
parser.add_argument("--psql-port", type=str, required=True, help="postgresql's port")
parser.add_argument("--psql-database", type=str, required=True, help="postgresql's database")
parser.add_argument("--psql-user", type=str, required=True, help="postgresql's user")
parser.add_argument("--psql-password", type=str, required=True, help="postgresql's password")
arguments = parser.parse_args()
return arguments
class PostgreSqlSettings:
def __init__(self):
self.host = None
self.port = None
self.database = None
self.user = None
self.password = None
class Settings:
def __init__(self):
self.host = None
self.port = None
self.postgresql_settings = None
def CreateSettings():
arguments = ParseArguments()
postgresql_settings = PostgreSqlSettings()
postgresql_settings.host = arguments.psql_host
postgresql_settings.port = arguments.psql_port
postgresql_settings.database = arguments.psql_database
postgresql_settings.user = arguments.psql_user
postgresql_settings.password = arguments.psql_password
settings = Settings()
settings.host = arguments.host
settings.port = arguments.port
settings.postgresql_settings = postgresql_settings
return settings
def LogCallback(level, message):
print(f"level: {level}, message: {message}")
def main():
settings = CreateSettings()
server = Server(settings.host, settings.port, Handle)
server.setLogger(LogLevel.Warning, LogCallback)
# Создаём контекст
context = CreateContext()
# Подключаемся к хранилищу
context.storage = Storage(
settings.postgresql_settings.host,
settings.postgresql_settings.port,
settings.postgresql_settings.database,
settings.postgresql_settings.user,
settings.postgresql_settings.password
)
# Заполняем пайплайны
query = "select name, pipes from pipelines;"
pipelines = context.storage.do(query, should_return=True)
for name, pipes in pipelines:
context.pipeline_manager.create(name, json.loads(pipes))
# Указываем контекст в сервере
server.context = context
# Запускаем сервер
asyncio.run(server.run())
if __name__ == "__main__":
main()
-62
View File
@@ -1,62 +0,0 @@
from modules.pipelines.pipe_manager import PipeManager
from modules.pipelines.pipeline_manager import PipelineManager
from modules.pipelines.pipes.preprocessing import *
from modules.handlers.handler_manager import HandlerManager
from modules.handlers.pipelines.create_pipeline import CreatePipeline
from modules.handlers.pipelines.get_pipelines import GetPipelines
from modules.handlers.pipelines.get_pipes import GetPipes
from modules.handlers.pipelines.run_pipeline import RunPipeline
from services.control.context import Context
def CreatePipeManager() -> PipeManager:
"""
Создание заполненного менеджера пайп
:return: Менеджер пайп
"""
manager = PipeManager()
manager.add(RemovePunctuationMarks())
manager.add(RemoveStopWords())
manager.add(Stem())
manager.add(ToLowerCase())
manager.add(SplitBySpace())
return manager
def CreatePipelineManager(pipe_manager) -> PipelineManager:
"""
Создание заполненного менеджера пайплайнов
:param pipe_manager: Менеджер пайп
:return: Менеджер пайплайнов
"""
manager = PipelineManager(pipe_manager)
return manager
def CreateHandlerManager() -> HandlerManager:
"""
Создание менеджера хэндлеров
:return: Менеджер хэндлеров
"""
manager = HandlerManager()
manager.add(CreatePipeline())
manager.add(GetPipelines())
manager.add(GetPipes())
manager.add(RunPipeline())
return manager
def CreateContext() -> Context:
"""
Создание заполненного контекста
:return: Контекст
"""
context = Context()
context.pipe_manager = CreatePipeManager()
context.pipeline_manager = CreatePipelineManager(context.pipe_manager)
context.handler_manager = CreateHandlerManager()
return context
-26
View File
@@ -1,26 +0,0 @@
import json
def SuccessResponse(result: str) -> str:
"""
Создание успешного ответа
:param result: Тело ответа
:return: Ответ
"""
return json.dumps({
"status": "ok",
"result": result
})
def ErrorResponse(status: str, message: str) -> str:
"""
Создание ответа с ошибкой
:param status: Код ошибки
:param message: Сообщение об ошибке
:return:
"""
return json.dumps({
"status": status,
"message": message
})
+41
View File
@@ -0,0 +1,41 @@
import re
from modules.logging.logger import LogLevel
from modules.utils.pattern import Constants, Pattern
from services.dispatchers.dispatcher_base import DispatcherBase
def ConsumeCallback(context, message):
match = re.search(str(context.pattern), message)
if match is None:
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
return
groups = match.groups()
topic = groups[5]
message = groups[6]
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
context.producer.send(topic, message)
class AllDispatcher(DispatcherBase):
def __init__(self):
# Паттерн: <number>month number hh:mm:ss machine-name process-name: message
pattern = Pattern()
pattern.parts = [
(Constants.unknown_begin_part, "unknown begin part"), # 0
(Constants.month, "month"), # 1
(Constants.number, "unknown number"), # 2
(Constants.time, "time"), # 3
(Constants.word, "machine name"), # 4
(Constants.word + ":", "process name"), # 5
(Constants.anything, "message") # 6
]
super().__init__(topic="all", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for all logs")
def main():
dispatcher = AllDispatcher()
dispatcher.run()
if __name__ == "__main__":
main()
@@ -0,0 +1,43 @@
import re
from modules.logging.logger import LogLevel
from modules.utils.pattern import Constants, Pattern
from services.dispatchers.dispatcher_base import DispatcherBase
def ConsumeCallback(context, message):
match = re.search(str(context.pattern), message)
if match is None:
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
return
groups = match.groups()
topic = groups[3]
message = groups[4]
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
context.producer.send(topic, message)
class CephOsdDispatcher(DispatcherBase):
def __init__(self):
# Паттерн: yyyy-mm-ddThh:mm:ss+timezone pid number process-name: message
datetime = Constants.number + "-" + Constants.number + "-" + Constants.number + "T" + Constants.time + "\." + Constants.number + "\+" + Constants.number
datetime = datetime.replace("(", "")
datetime = datetime.replace(")", "")
pattern = Pattern()
pattern.parts = [
("(" + datetime + ")", "Date and time with timezone"), # 0
(Constants.word, "pid"), # 1
(Constants.number, "unknown number"), # 2
(Constants.word + ":", "process name"), # 3
(Constants.anything, "message") # 4
]
super().__init__(topic="ceph-osd", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for ceph-osd logs")
def main():
dispatcher = CephOsdDispatcher()
dispatcher.run()
if __name__ == "__main__":
main()
+51
View File
@@ -0,0 +1,51 @@
import argparse
from datetime import datetime
from modules.logging.logger import LogLevel, Logger
from modules.message_broker.consumer import Config as ConsumerConfig, Consumer
from modules.message_broker.producer import Config as ProducerConfig, Producer
from modules.utils.pattern import Pattern
class Context:
def __init__(self):
self.logger = None
self.pattern = None
self.producer = None
def LogCallback(level, message):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{str(level)}] {now}: {message}")
def ParseArguments(description):
parser = argparse.ArgumentParser(description=description)
parser.add_argument("--server", type=str, required=True, help="Kafka server")
parser.add_argument("--pool-size", type=int, required=True, help="Pool size")
parser.add_argument("--pool-cache-limit", type=int, required=True, help="Pool cache limit")
parser.add_argument("--max-log-level", type=int, required=False, help="Max log level. Error -- 0 (default), Warning -- 1, Debug -- 2")
arguments = parser.parse_args()
return arguments
class DispatcherBase:
def __init__(self, topic: str, pattern: Pattern, consume_callback, description: str):
# Парсим аргументы
arguments = ParseArguments(description)
# Создаём контекст
self.context = Context()
self.context.pattern = pattern
# Настраиваем логирование
max_log_level = LogLevel.Debug # todo log level
self.context.logger = Logger(max_log_level=max_log_level, callback=LogCallback)
# Создаём продюсера
producer_config = ProducerConfig(arguments.server)
self.context.producer = Producer(producer_config)
# Создаём консюмера
consumer_config = ConsumerConfig(topic, arguments.server, arguments.pool_size, arguments.pool_cache_limit)
self.consumer = Consumer(consumer_config, consume_callback)
def run(self):
self.consumer.run(self.context)
+4
View File
@@ -0,0 +1,4 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+138
View File
@@ -0,0 +1,138 @@
import argparse
import json
import re
from modules.message_broker.consumer import Config, Consumer
from modules.storages.storage import ConnectionData, Storage
from modules.utils.pattern import Constants, Pattern
from services.log_analyzer.model import LogAnalyzer
class Context:
def __init__(self):
self.kafka_server = None
self.topic = None
self.model = None
self.storage = None
self.message_pattern = None
self.enable_logging = False
def CheckRequiredKeys(config):
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
is_ok = True
for key in keys:
if key not in config:
print(f"config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
storage_config = config["storage"]
storage_keys = ["host", "port", "database", "user", "password"]
for key in storage_keys:
if key not in storage_config:
print(f"storage config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
def ParseConfig() -> Context:
parser = argparse.ArgumentParser("Log analyzer")
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
arguments = parser.parse_args()
with open(arguments.config, "r") as file:
config_data = file.read()
config = json.loads(config_data)
CheckRequiredKeys(config)
context = Context()
context.kafka_server = config["kafka_server"]
context.topic = config["topic"]
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
context.model.fit(config["model"])
storage_config = config["storage"]
connection_data = ConnectionData()
connection_data.host = storage_config["host"]
connection_data.port = storage_config["port"]
connection_data.database = storage_config["database"]
connection_data.user = storage_config["user"]
connection_data.password = storage_config["password"]
context.storage = Storage(connection_data)
context.enable_logging = arguments.enable_logging
pattern = Pattern()
pattern.parts = [
(Constants.unknown_begin_part, "unknown begin part"), # 0
(Constants.month, "month"), # 1
(Constants.number, "unknown number"), # 2
(Constants.time, "time"), # 3
(Constants.word, "machine name"), # 4
(Constants.word + ":", "process name"), # 5
(Constants.anything, "message") # 6
]
context.message_pattern = re.compile(str(pattern))
return context
def ClearMessage(context: Context, message: str) -> tuple:
match = re.search(context.message_pattern, message)
if match is None:
if context.enable_logging:
print(f"Nothing is matched for log '{message}'")
return "", message
groups = match.groups()
topic = groups[5]
message = groups[6]
return topic, message
def UpdateStatisticsQuery(name: str):
return f"update statistics set value = value + 1 where name = '{name}';"
def SaveAnomalyLogQuery(log: str):
return f"insert into anomaly_logs (data) values('{log}')"
def Callback(context: Context, message: str):
if context.enable_logging:
print(f"Processing message '{message}'", message)
topic, message = ClearMessage(context, message)
result = context.model.predict(program=topic, message=message)[0]
result_as_str = "anomaly" if result == -1 else "not anomaly"
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
context.storage.do(query)
if result == -1:
query = SaveAnomalyLogQuery(message)
context.storage.do(query)
if context.enable_logging:
print("log is " + result_as_str)
def main():
# Создаём контекст
context = ParseConfig()
# Создаём потребителя
config = Config(context.topic, context.kafka_server, 16, 16)
consumer = Consumer(config, Callback)
# Запускаем обработку
consumer.run(context)
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
from transformers import AutoTokenizer, AutoModel, AutoConfig
import torch
import torch.nn.functional as F
import pickle
import numpy
import pandas as pd
class LogAnalyzer:
'''Сlass of an expert system module that analyzes diagnostic information
about the operation of the input-output subsystem.
'''
def __init__(self, path_model_embedding: str, path_tokenizer: str):
'''Constructor of model for analyse.
:param path_model_embedding: String of path to directory with model
for build embeddings of message
:param path_tokenizer: String of path to directory with tokenizer
for tokenize message
'''
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
self.model = None
def predict(self, program: str, message: str) -> numpy.ndarray:
'''Function predict whether an log is anomaly.
:param message: string of message
'''
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
'''Function applies mean pooling on the embedding model to get
a single vector representation.
:param model_output: Tuple containing different model outputs.
The first element contains the token embeddings.
:param attention_mask: Tensor that identifies the tokens
to which attention should be paid (value of 1)
versus those which should not (value of 0).
:return: Tensor representing the pooled output of the input tokens.
'''
token_embeddings = model_output[0]
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
token_embeddings.size()).float()
return torch.sum(
token_embeddings * input_mask_expanded, 1) / torch.clamp(
input_mask_expanded.sum(1), min=1e-9)
def embed(message: str, tokenizer, model) -> torch.Tensor:
'''Function generates embeddings for a given message.
:param message: String of message for which embeddings
are to be generated.
:param tokenizer: The tokenizer to be used for tokenizing
the message.
:param model: The model to be used for generating embeddings.
:return: Tensor of normalized embeddings of the input message.
'''
encoded_input = tokenizer(
message, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
model_output = model(**encoded_input)
message_embeddings_mean = mean_pooling(
model_output, encoded_input['attention_mask'])
message_embeddings = F.normalize(
message_embeddings_mean, p=2, dim=1)
return message_embeddings
if self.model is None:
raise ValueError("Model is not fitted. Make '.fit' method first")
return None
message_embeddings = embed(
message, self.tokenizer, self.model_embedding)
try:
prediction = self.model[program].predict(
message_embeddings.numpy())
except KeyError:
prediction = self.model['unknown'].predict(
message_embeddings.numpy())
return prediction
def fit(self, path_model):
'''Function predict whether an log is anomaly.
:param path_model: String of path to saved model
'''
self.model = pickle.load(open(path_model, 'rb'))
+33
View File
@@ -0,0 +1,33 @@
import re
from modules.pipelines.pipes.preprocessing import DataType, PipeBase
class RemoveByRegex(PipeBase):
def __init__(self, name, pattern):
super().__init__(name, DataType.String, DataType.String)
self.pattern = re.compile(pattern)
def do(self, data):
return self.pattern.sub("", data)
class RemoveOriginalLogTime(RemoveByRegex):
def __init__(self):
super().__init__("remove_original_log_time", "\(Original Log Time .*\)")
class RemoveSourceCode(RemoveByRegex):
def __init__(self):
super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]")
class StopRunningIfContains(PipeBase):
def __init__(self, items: list[str]):
super().__init__("stop_running_if_contains", DataType.String, DataType.String)
self.items = items
def do(self, data: str):
for item in self.items:
if data.find(item) != -1:
return None
return data
@@ -0,0 +1,22 @@
from modules.pipelines.pipeline import Pipeline
from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace
from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains
def main():
pipeline = Pipeline("prepare")
pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"]))
pipeline.add(RemoveOriginalLogTime())
pipeline.add(RemoveSourceCode())
pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]"))
pipeline.add(Strip())
with open("rocksdb_logs.txt", "r") as input:
with open("filtered_rocksdb_logs.txt", "w") as output:
for line in input.readlines():
result = pipeline.run(line)
if result is not None:
output.write(result + "\n")
if __name__ == "__main__":
main()