Большое обновление всего проекта
This commit is contained in:
@@ -1,193 +0,0 @@
|
||||
import json
|
||||
|
||||
from modules.transports.client import Client
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
|
||||
class CommandException(Exception):
|
||||
def __init__(self, message):
|
||||
super().__init__(message)
|
||||
|
||||
class Command:
|
||||
def __init__(self, name):
|
||||
"""
|
||||
Конструктор
|
||||
:param name: Название команды
|
||||
"""
|
||||
self.name = name
|
||||
self.settings = None
|
||||
|
||||
def do(self, parameters):
|
||||
"""
|
||||
Вызов команды
|
||||
:param parameters: Параметры
|
||||
:return: Результат работы
|
||||
"""
|
||||
raise Exception("calling unimplemented method")
|
||||
|
||||
def doRequest(self, request: dict):
|
||||
"""
|
||||
Выполнение запроса
|
||||
:param request: Тело запроса
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
# Создаём клиентский сокет и подключаемся к серверу
|
||||
client = Client()
|
||||
client.connect(self.settings.host, self.settings.port)
|
||||
|
||||
# Отправляем запрос
|
||||
client.send(json.dumps(request))
|
||||
|
||||
# Получаем ответ
|
||||
response = client.receive()
|
||||
|
||||
# Закрываем соединение
|
||||
client.disconnect()
|
||||
|
||||
# Парсим ответ
|
||||
response = json.loads(response)
|
||||
|
||||
# Проверяем статус обработки запроса
|
||||
status = response["status"]
|
||||
if status != "ok":
|
||||
message = response["message"]
|
||||
raise CommandException(f"status: {status}, message: {message}")
|
||||
|
||||
return response["result"]
|
||||
|
||||
class CommandManager:
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.commands = []
|
||||
self.command_name_to_index = {}
|
||||
|
||||
def addCommand(self, command: Command):
|
||||
command.settings = self.settings
|
||||
self.command_name_to_index[command.name] = len(self.commands)
|
||||
self.commands.append(command)
|
||||
|
||||
def hasCommand(self, name) -> bool:
|
||||
if str.isnumeric(name):
|
||||
index = int(name)
|
||||
if index < 0 or index >= len(self.commands):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
return name in self.command_name_to_index
|
||||
|
||||
def getCommand(self, name):
|
||||
if not self.hasCommand(name):
|
||||
raise Exception("there is no command with such name")
|
||||
|
||||
if str.isnumeric(name):
|
||||
return self.commands[int(name)]
|
||||
|
||||
return self.commands[self.command_name_to_index[name]]
|
||||
|
||||
class GetPipes(Command):
|
||||
"""
|
||||
Получение списка пайп
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, parameters):
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": ""
|
||||
}
|
||||
|
||||
# Ответ -- это список пайп
|
||||
response = self.doRequest(request)
|
||||
|
||||
if len(response) == 0:
|
||||
print("there are no pipes")
|
||||
return
|
||||
|
||||
for pipe in response[:-1]:
|
||||
self.printPipe(pipe)
|
||||
print()
|
||||
|
||||
self.printPipe(response[-1])
|
||||
|
||||
def printPipe(self, pipe):
|
||||
print(pipe["name"] + ":")
|
||||
print("- description: " + pipe["description"])
|
||||
print("- input data type: " + pipe["input_data_type"])
|
||||
print("- output data type: " + pipe["output_data_type"])
|
||||
|
||||
class GetPipelines(Command):
|
||||
"""
|
||||
Получение списка пайплайнов
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, parameters):
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": ""
|
||||
}
|
||||
|
||||
# Ответ -- это список пайплайнов
|
||||
response = self.doRequest(request)
|
||||
|
||||
if len(response) == 0:
|
||||
print("there are no pipelines")
|
||||
return
|
||||
|
||||
for pipeline in response:
|
||||
print(pipeline["name"] + ": " + str(pipeline["pipes"]))
|
||||
|
||||
class RunPipeline(Command):
|
||||
"""
|
||||
Запуск пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, parameters):
|
||||
pipeline = input("pipeline: ")
|
||||
data = input("data: ")
|
||||
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": {
|
||||
"name": pipeline,
|
||||
"data": data
|
||||
}
|
||||
}
|
||||
|
||||
response = self.doRequest(request)
|
||||
print("result: " + str(response))
|
||||
|
||||
class CreatePipeline(Command):
|
||||
"""
|
||||
Создание пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, parameters):
|
||||
name = input("name: ")
|
||||
pipes = input("pipes (separated by spaces): ")
|
||||
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": {
|
||||
"name": name,
|
||||
"pipes": pipes.split()
|
||||
}
|
||||
}
|
||||
|
||||
self.doRequest(request)
|
||||
print("created successfully")
|
||||
@@ -1,70 +0,0 @@
|
||||
import argparse
|
||||
|
||||
from services.console_client.commands import *
|
||||
|
||||
def CreateCommandManager(settings):
|
||||
manager = CommandManager(settings)
|
||||
manager.addCommand(GetPipes())
|
||||
manager.addCommand(GetPipelines())
|
||||
manager.addCommand(CreatePipeline())
|
||||
manager.addCommand(RunPipeline())
|
||||
|
||||
return manager
|
||||
|
||||
def PrintHelp(command_manager):
|
||||
for index, command in enumerate(command_manager.commands):
|
||||
print(str(index) + ". " + command.name)
|
||||
|
||||
def ParseArguments():
|
||||
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
|
||||
|
||||
# Хост сервера
|
||||
parser.add_argument("--host", type=str, required=True, help="server's host")
|
||||
|
||||
# Порт
|
||||
parser.add_argument("--port", type=int, required=True, help="server's port")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
def CreateSettings() -> Settings:
|
||||
arguments = ParseArguments()
|
||||
|
||||
settings = Settings()
|
||||
settings.host = arguments.host
|
||||
settings.port = arguments.port
|
||||
|
||||
return settings
|
||||
|
||||
def main():
|
||||
settings = CreateSettings()
|
||||
command_manager = CreateCommandManager(settings)
|
||||
|
||||
parameters = {}
|
||||
|
||||
PrintHelp(command_manager)
|
||||
|
||||
while True:
|
||||
command_name = str(input("> "))
|
||||
if command_name in ["q", "quit", "exit"]:
|
||||
break
|
||||
|
||||
if command_name in ["h", "help"]:
|
||||
PrintHelp(command_manager)
|
||||
continue
|
||||
|
||||
if not command_manager.hasCommand(command_name):
|
||||
print("there is no command with such name")
|
||||
continue
|
||||
|
||||
command = command_manager.getCommand(command_name)
|
||||
|
||||
try:
|
||||
command.do(parameters)
|
||||
except Exception as e:
|
||||
print(f"error: {str(e)}")
|
||||
|
||||
print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,19 +0,0 @@
|
||||
from modules.pipelines.pipe_manager import PipeManager
|
||||
from modules.pipelines.pipeline_manager import PipelineManager
|
||||
from modules.handlers.handler_manager import HandlerManager
|
||||
from modules.storages.storage import Storage
|
||||
|
||||
class Context:
|
||||
"""
|
||||
Контекст управляющего сервиса
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
self.pipe_manager = None
|
||||
self.pipeline_manager = None
|
||||
self.handler_manager = None
|
||||
self.storage = None
|
||||
@@ -1,130 +0,0 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from modules.logging.logger import *
|
||||
from modules.storages.storage import Storage
|
||||
from modules.transports.server import Server
|
||||
from services.control.context import *
|
||||
from services.control.create_context import *
|
||||
from services.control.response import *
|
||||
|
||||
def Handle(context, request):
|
||||
"""
|
||||
Обработка запроса
|
||||
:param context: Контекст сервиса
|
||||
:param request: Запрос
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
# Парсим запрос
|
||||
try:
|
||||
request = json.loads(request)
|
||||
except Exception as e:
|
||||
return ErrorResponse("invalid request", "failed to parse request")
|
||||
|
||||
# Проверяем наличие нужных ключей
|
||||
for key in ["handler", "parameters"]:
|
||||
if key not in request:
|
||||
return ErrorResponse("invalid request", f"key '{key}' is required")
|
||||
|
||||
# Проверяем наличие обработчика с указанным именем
|
||||
handler_name = request["handler"]
|
||||
if not context.handler_manager.contains(handler_name):
|
||||
return ErrorResponse("not found", f"there is no handler with name {handler_name}")
|
||||
|
||||
# Достаём нужного обработчика
|
||||
handler = context.handler_manager.get(handler_name)
|
||||
|
||||
# Запускаем обработку
|
||||
try:
|
||||
result = handler.do(context, request["parameters"])
|
||||
return SuccessResponse(result)
|
||||
except Exception as e:
|
||||
return ErrorResponse("handling failed", str(e))
|
||||
|
||||
|
||||
def ParseArguments():
|
||||
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
|
||||
|
||||
# Данные сервера
|
||||
parser.add_argument("--host", type=str, required=True, help="server's host")
|
||||
parser.add_argument("--port", type=int, required=True, help="server's port")
|
||||
|
||||
# Данные для подключения к базе данных
|
||||
parser.add_argument("--psql-host", type=str, required=True, help="postgresql's host")
|
||||
parser.add_argument("--psql-port", type=str, required=True, help="postgresql's port")
|
||||
parser.add_argument("--psql-database", type=str, required=True, help="postgresql's database")
|
||||
parser.add_argument("--psql-user", type=str, required=True, help="postgresql's user")
|
||||
parser.add_argument("--psql-password", type=str, required=True, help="postgresql's password")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
class PostgreSqlSettings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.database = None
|
||||
self.user = None
|
||||
self.password = None
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.postgresql_settings = None
|
||||
|
||||
def CreateSettings():
|
||||
arguments = ParseArguments()
|
||||
|
||||
postgresql_settings = PostgreSqlSettings()
|
||||
postgresql_settings.host = arguments.psql_host
|
||||
postgresql_settings.port = arguments.psql_port
|
||||
postgresql_settings.database = arguments.psql_database
|
||||
postgresql_settings.user = arguments.psql_user
|
||||
postgresql_settings.password = arguments.psql_password
|
||||
|
||||
settings = Settings()
|
||||
settings.host = arguments.host
|
||||
settings.port = arguments.port
|
||||
settings.postgresql_settings = postgresql_settings
|
||||
|
||||
return settings
|
||||
|
||||
def LogCallback(level, message):
|
||||
print(f"level: {level}, message: {message}")
|
||||
|
||||
def main():
|
||||
settings = CreateSettings()
|
||||
|
||||
server = Server(settings.host, settings.port, Handle)
|
||||
server.setLogger(LogLevel.Warning, LogCallback)
|
||||
|
||||
# Создаём контекст
|
||||
context = CreateContext()
|
||||
|
||||
# Подключаемся к хранилищу
|
||||
context.storage = Storage(
|
||||
settings.postgresql_settings.host,
|
||||
settings.postgresql_settings.port,
|
||||
settings.postgresql_settings.database,
|
||||
settings.postgresql_settings.user,
|
||||
settings.postgresql_settings.password
|
||||
)
|
||||
|
||||
# Заполняем пайплайны
|
||||
query = "select name, pipes from pipelines;"
|
||||
pipelines = context.storage.do(query, should_return=True)
|
||||
|
||||
for name, pipes in pipelines:
|
||||
context.pipeline_manager.create(name, json.loads(pipes))
|
||||
|
||||
# Указываем контекст в сервере
|
||||
server.context = context
|
||||
|
||||
# Запускаем сервер
|
||||
asyncio.run(server.run())
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,62 +0,0 @@
|
||||
from modules.pipelines.pipe_manager import PipeManager
|
||||
from modules.pipelines.pipeline_manager import PipelineManager
|
||||
from modules.pipelines.pipes.preprocessing import *
|
||||
|
||||
from modules.handlers.handler_manager import HandlerManager
|
||||
from modules.handlers.pipelines.create_pipeline import CreatePipeline
|
||||
from modules.handlers.pipelines.get_pipelines import GetPipelines
|
||||
from modules.handlers.pipelines.get_pipes import GetPipes
|
||||
from modules.handlers.pipelines.run_pipeline import RunPipeline
|
||||
|
||||
from services.control.context import Context
|
||||
|
||||
def CreatePipeManager() -> PipeManager:
|
||||
"""
|
||||
Создание заполненного менеджера пайп
|
||||
:return: Менеджер пайп
|
||||
"""
|
||||
|
||||
manager = PipeManager()
|
||||
manager.add(RemovePunctuationMarks())
|
||||
manager.add(RemoveStopWords())
|
||||
manager.add(Stem())
|
||||
manager.add(ToLowerCase())
|
||||
manager.add(SplitBySpace())
|
||||
|
||||
return manager
|
||||
|
||||
def CreatePipelineManager(pipe_manager) -> PipelineManager:
|
||||
"""
|
||||
Создание заполненного менеджера пайплайнов
|
||||
:param pipe_manager: Менеджер пайп
|
||||
:return: Менеджер пайплайнов
|
||||
"""
|
||||
|
||||
manager = PipelineManager(pipe_manager)
|
||||
return manager
|
||||
|
||||
def CreateHandlerManager() -> HandlerManager:
|
||||
"""
|
||||
Создание менеджера хэндлеров
|
||||
:return: Менеджер хэндлеров
|
||||
"""
|
||||
manager = HandlerManager()
|
||||
manager.add(CreatePipeline())
|
||||
manager.add(GetPipelines())
|
||||
manager.add(GetPipes())
|
||||
manager.add(RunPipeline())
|
||||
|
||||
return manager
|
||||
|
||||
def CreateContext() -> Context:
|
||||
"""
|
||||
Создание заполненного контекста
|
||||
:return: Контекст
|
||||
"""
|
||||
|
||||
context = Context()
|
||||
context.pipe_manager = CreatePipeManager()
|
||||
context.pipeline_manager = CreatePipelineManager(context.pipe_manager)
|
||||
context.handler_manager = CreateHandlerManager()
|
||||
|
||||
return context
|
||||
@@ -1,26 +0,0 @@
|
||||
import json
|
||||
|
||||
def SuccessResponse(result: str) -> str:
|
||||
"""
|
||||
Создание успешного ответа
|
||||
:param result: Тело ответа
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
return json.dumps({
|
||||
"status": "ok",
|
||||
"result": result
|
||||
})
|
||||
|
||||
def ErrorResponse(status: str, message: str) -> str:
|
||||
"""
|
||||
Создание ответа с ошибкой
|
||||
:param status: Код ошибки
|
||||
:param message: Сообщение об ошибке
|
||||
:return:
|
||||
"""
|
||||
|
||||
return json.dumps({
|
||||
"status": status,
|
||||
"message": message
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import re
|
||||
|
||||
from modules.logging.logger import LogLevel
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.dispatchers.dispatcher_base import DispatcherBase
|
||||
|
||||
def ConsumeCallback(context, message):
|
||||
match = re.search(str(context.pattern), message)
|
||||
if match is None:
|
||||
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
|
||||
return
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
|
||||
context.producer.send(topic, message)
|
||||
|
||||
class AllDispatcher(DispatcherBase):
|
||||
def __init__(self):
|
||||
# Паттерн: <number>month number hh:mm:ss machine-name process-name: message
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
super().__init__(topic="all", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for all logs")
|
||||
|
||||
def main():
|
||||
dispatcher = AllDispatcher()
|
||||
dispatcher.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
import re
|
||||
|
||||
from modules.logging.logger import LogLevel
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.dispatchers.dispatcher_base import DispatcherBase
|
||||
|
||||
def ConsumeCallback(context, message):
|
||||
match = re.search(str(context.pattern), message)
|
||||
if match is None:
|
||||
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
|
||||
return
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[3]
|
||||
message = groups[4]
|
||||
|
||||
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
|
||||
context.producer.send(topic, message)
|
||||
|
||||
class CephOsdDispatcher(DispatcherBase):
|
||||
def __init__(self):
|
||||
# Паттерн: yyyy-mm-ddThh:mm:ss+timezone pid number process-name: message
|
||||
datetime = Constants.number + "-" + Constants.number + "-" + Constants.number + "T" + Constants.time + "\." + Constants.number + "\+" + Constants.number
|
||||
datetime = datetime.replace("(", "")
|
||||
datetime = datetime.replace(")", "")
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
("(" + datetime + ")", "Date and time with timezone"), # 0
|
||||
(Constants.word, "pid"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.word + ":", "process name"), # 3
|
||||
(Constants.anything, "message") # 4
|
||||
]
|
||||
|
||||
super().__init__(topic="ceph-osd", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for ceph-osd logs")
|
||||
|
||||
def main():
|
||||
dispatcher = CephOsdDispatcher()
|
||||
dispatcher.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
from modules.logging.logger import LogLevel, Logger
|
||||
from modules.message_broker.consumer import Config as ConsumerConfig, Consumer
|
||||
from modules.message_broker.producer import Config as ProducerConfig, Producer
|
||||
from modules.utils.pattern import Pattern
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.logger = None
|
||||
self.pattern = None
|
||||
self.producer = None
|
||||
|
||||
def LogCallback(level, message):
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f"[{str(level)}] {now}: {message}")
|
||||
|
||||
def ParseArguments(description):
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument("--server", type=str, required=True, help="Kafka server")
|
||||
parser.add_argument("--pool-size", type=int, required=True, help="Pool size")
|
||||
parser.add_argument("--pool-cache-limit", type=int, required=True, help="Pool cache limit")
|
||||
parser.add_argument("--max-log-level", type=int, required=False, help="Max log level. Error -- 0 (default), Warning -- 1, Debug -- 2")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
class DispatcherBase:
|
||||
def __init__(self, topic: str, pattern: Pattern, consume_callback, description: str):
|
||||
# Парсим аргументы
|
||||
arguments = ParseArguments(description)
|
||||
|
||||
# Создаём контекст
|
||||
self.context = Context()
|
||||
self.context.pattern = pattern
|
||||
|
||||
# Настраиваем логирование
|
||||
max_log_level = LogLevel.Debug # todo log level
|
||||
self.context.logger = Logger(max_log_level=max_log_level, callback=LogCallback)
|
||||
|
||||
# Создаём продюсера
|
||||
producer_config = ProducerConfig(arguments.server)
|
||||
self.context.producer = Producer(producer_config)
|
||||
|
||||
# Создаём консюмера
|
||||
consumer_config = ConsumerConfig(topic, arguments.server, arguments.pool_size, arguments.pool_cache_limit)
|
||||
self.consumer = Consumer(consumer_config, consume_callback)
|
||||
|
||||
def run(self):
|
||||
self.consumer.run(self.context)
|
||||
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
@@ -0,0 +1,138 @@
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
|
||||
from modules.message_broker.consumer import Config, Consumer
|
||||
from modules.storages.storage import ConnectionData, Storage
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.log_analyzer.model import LogAnalyzer
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.kafka_server = None
|
||||
self.topic = None
|
||||
self.model = None
|
||||
self.storage = None
|
||||
self.message_pattern = None
|
||||
self.enable_logging = False
|
||||
|
||||
def CheckRequiredKeys(config):
|
||||
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
|
||||
|
||||
is_ok = True
|
||||
for key in keys:
|
||||
if key not in config:
|
||||
print(f"config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
storage_config = config["storage"]
|
||||
storage_keys = ["host", "port", "database", "user", "password"]
|
||||
for key in storage_keys:
|
||||
if key not in storage_config:
|
||||
print(f"storage config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
def ParseConfig() -> Context:
|
||||
parser = argparse.ArgumentParser("Log analyzer")
|
||||
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
|
||||
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
with open(arguments.config, "r") as file:
|
||||
config_data = file.read()
|
||||
|
||||
config = json.loads(config_data)
|
||||
CheckRequiredKeys(config)
|
||||
|
||||
context = Context()
|
||||
context.kafka_server = config["kafka_server"]
|
||||
context.topic = config["topic"]
|
||||
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
|
||||
context.model.fit(config["model"])
|
||||
|
||||
storage_config = config["storage"]
|
||||
connection_data = ConnectionData()
|
||||
connection_data.host = storage_config["host"]
|
||||
connection_data.port = storage_config["port"]
|
||||
connection_data.database = storage_config["database"]
|
||||
connection_data.user = storage_config["user"]
|
||||
connection_data.password = storage_config["password"]
|
||||
|
||||
context.storage = Storage(connection_data)
|
||||
|
||||
context.enable_logging = arguments.enable_logging
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
context.message_pattern = re.compile(str(pattern))
|
||||
|
||||
return context
|
||||
|
||||
def ClearMessage(context: Context, message: str) -> tuple:
|
||||
match = re.search(context.message_pattern, message)
|
||||
if match is None:
|
||||
if context.enable_logging:
|
||||
print(f"Nothing is matched for log '{message}'")
|
||||
|
||||
return "", message
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
return topic, message
|
||||
|
||||
def UpdateStatisticsQuery(name: str):
|
||||
return f"update statistics set value = value + 1 where name = '{name}';"
|
||||
|
||||
def SaveAnomalyLogQuery(log: str):
|
||||
return f"insert into anomaly_logs (data) values('{log}')"
|
||||
|
||||
def Callback(context: Context, message: str):
|
||||
if context.enable_logging:
|
||||
print(f"Processing message '{message}'", message)
|
||||
|
||||
topic, message = ClearMessage(context, message)
|
||||
|
||||
result = context.model.predict(program=topic, message=message)[0]
|
||||
result_as_str = "anomaly" if result == -1 else "not anomaly"
|
||||
|
||||
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
|
||||
context.storage.do(query)
|
||||
|
||||
if result == -1:
|
||||
query = SaveAnomalyLogQuery(message)
|
||||
context.storage.do(query)
|
||||
|
||||
if context.enable_logging:
|
||||
print("log is " + result_as_str)
|
||||
|
||||
def main():
|
||||
# Создаём контекст
|
||||
context = ParseConfig()
|
||||
|
||||
# Создаём потребителя
|
||||
config = Config(context.topic, context.kafka_server, 16, 16)
|
||||
consumer = Consumer(config, Callback)
|
||||
|
||||
# Запускаем обработку
|
||||
consumer.run(context)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
from transformers import AutoTokenizer, AutoModel, AutoConfig
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import pickle
|
||||
import numpy
|
||||
import pandas as pd
|
||||
|
||||
class LogAnalyzer:
|
||||
'''Сlass of an expert system module that analyzes diagnostic information
|
||||
about the operation of the input-output subsystem.
|
||||
'''
|
||||
|
||||
def __init__(self, path_model_embedding: str, path_tokenizer: str):
|
||||
'''Constructor of model for analyse.
|
||||
:param path_model_embedding: String of path to directory with model
|
||||
for build embeddings of message
|
||||
:param path_tokenizer: String of path to directory with tokenizer
|
||||
for tokenize message
|
||||
'''
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
|
||||
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
|
||||
self.model = None
|
||||
|
||||
def predict(self, program: str, message: str) -> numpy.ndarray:
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param message: string of message
|
||||
'''
|
||||
|
||||
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
|
||||
'''Function applies mean pooling on the embedding model to get
|
||||
a single vector representation.
|
||||
:param model_output: Tuple containing different model outputs.
|
||||
The first element contains the token embeddings.
|
||||
:param attention_mask: Tensor that identifies the tokens
|
||||
to which attention should be paid (value of 1)
|
||||
versus those which should not (value of 0).
|
||||
:return: Tensor representing the pooled output of the input tokens.
|
||||
'''
|
||||
token_embeddings = model_output[0]
|
||||
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
|
||||
token_embeddings.size()).float()
|
||||
return torch.sum(
|
||||
token_embeddings * input_mask_expanded, 1) / torch.clamp(
|
||||
input_mask_expanded.sum(1), min=1e-9)
|
||||
|
||||
def embed(message: str, tokenizer, model) -> torch.Tensor:
|
||||
'''Function generates embeddings for a given message.
|
||||
:param message: String of message for which embeddings
|
||||
are to be generated.
|
||||
:param tokenizer: The tokenizer to be used for tokenizing
|
||||
the message.
|
||||
:param model: The model to be used for generating embeddings.
|
||||
:return: Tensor of normalized embeddings of the input message.
|
||||
'''
|
||||
encoded_input = tokenizer(
|
||||
message, padding=True, truncation=True, return_tensors='pt')
|
||||
|
||||
with torch.no_grad():
|
||||
model_output = model(**encoded_input)
|
||||
|
||||
message_embeddings_mean = mean_pooling(
|
||||
model_output, encoded_input['attention_mask'])
|
||||
message_embeddings = F.normalize(
|
||||
message_embeddings_mean, p=2, dim=1)
|
||||
return message_embeddings
|
||||
|
||||
if self.model is None:
|
||||
raise ValueError("Model is not fitted. Make '.fit' method first")
|
||||
return None
|
||||
|
||||
message_embeddings = embed(
|
||||
message, self.tokenizer, self.model_embedding)
|
||||
|
||||
try:
|
||||
prediction = self.model[program].predict(
|
||||
message_embeddings.numpy())
|
||||
except KeyError:
|
||||
prediction = self.model['unknown'].predict(
|
||||
message_embeddings.numpy())
|
||||
return prediction
|
||||
|
||||
def fit(self, path_model):
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param path_model: String of path to saved model
|
||||
'''
|
||||
self.model = pickle.load(open(path_model, 'rb'))
|
||||
@@ -0,0 +1,33 @@
|
||||
import re
|
||||
|
||||
from modules.pipelines.pipes.preprocessing import DataType, PipeBase
|
||||
|
||||
class RemoveByRegex(PipeBase):
|
||||
def __init__(self, name, pattern):
|
||||
super().__init__(name, DataType.String, DataType.String)
|
||||
|
||||
self.pattern = re.compile(pattern)
|
||||
|
||||
def do(self, data):
|
||||
return self.pattern.sub("", data)
|
||||
|
||||
class RemoveOriginalLogTime(RemoveByRegex):
|
||||
def __init__(self):
|
||||
super().__init__("remove_original_log_time", "\(Original Log Time .*\)")
|
||||
|
||||
class RemoveSourceCode(RemoveByRegex):
|
||||
def __init__(self):
|
||||
super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]")
|
||||
|
||||
class StopRunningIfContains(PipeBase):
|
||||
def __init__(self, items: list[str]):
|
||||
super().__init__("stop_running_if_contains", DataType.String, DataType.String)
|
||||
|
||||
self.items = items
|
||||
|
||||
def do(self, data: str):
|
||||
for item in self.items:
|
||||
if data.find(item) != -1:
|
||||
return None
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,22 @@
|
||||
from modules.pipelines.pipeline import Pipeline
|
||||
from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace
|
||||
from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains
|
||||
|
||||
def main():
|
||||
pipeline = Pipeline("prepare")
|
||||
pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"]))
|
||||
pipeline.add(RemoveOriginalLogTime())
|
||||
pipeline.add(RemoveSourceCode())
|
||||
pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]"))
|
||||
pipeline.add(Strip())
|
||||
|
||||
with open("rocksdb_logs.txt", "r") as input:
|
||||
with open("filtered_rocksdb_logs.txt", "w") as output:
|
||||
for line in input.readlines():
|
||||
result = pipeline.run(line)
|
||||
|
||||
if result is not None:
|
||||
output.write(result + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user