Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+24
View File
@@ -0,0 +1,24 @@
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ports:
- 22181:2181
kafka:
image: confluentinc/cp-kafka:latest
depends_on:
- zookeeper
ports:
- 29092:29092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+18
View File
@@ -0,0 +1,18 @@
class Subscriber:
def __init__(self):
self.host = None
self.port = None
class Context:
"""
Контекст обработчиков
"""
def __init__(self):
"""
Конструктор
"""
self.pipe_manager = None
self.pipeline_manager = None
self.subscribers = {}
@@ -1,3 +1,5 @@
from modules.handlers.context import Context
def CheckParameters(parameters, required_keys):
for key in required_keys:
if key not in parameters:
@@ -7,5 +9,5 @@ class HandlerBase:
def __init__(self, name):
self.name = name
def do(self, context, parameters):
def do(self, context: Context, parameters: dict):
pass
+109
View File
@@ -0,0 +1,109 @@
import json
from modules.pipelines.pipes.pipe_base import ToString
from modules.handlers.handler_base import *
class CreatePipeline(HandlerBase):
"""
Обработчик создания пайплайна
"""
def __init__(self):
super().__init__("create_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "pipes"])
name = parameters["name"]
pipes = parameters["pipes"]
context.pipeline_manager.create(name, pipes)
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
context.storage.do(query)
class GetPipelines(HandlerBase):
"""
Обработчик получения списка пайплайнов
"""
def __init__(self):
super().__init__("get_pipelines")
def do(self, context: Context, parameters: dict):
result = []
for name, pipeline in context.pipeline_manager.pipelines.items():
item = {
"name": pipeline.name,
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
}
result.append(item)
return result
class GetPipes(HandlerBase):
"""
Обработчик получения списка пайп
"""
def __init__(self):
super().__init__("get_pipes")
def do(self, context: Context, parameters: dict):
result = []
for pipe_name, pipe in context.pipe_manager.pipes.items():
item = {
"name": pipe.name,
"description": pipe.description,
"input_data_type": ToString(pipe.input_data_type),
"output_data_type": ToString(pipe.output_data_type)
}
result.append(item)
return result
class RunPipeline(HandlerBase):
"""
Обработчик запуска пайплайна
"""
def __init__(self):
super().__init__("run_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "data"])
name = parameters["name"]
data = parameters["data"]
pipeline = context.pipeline_manager.get(name)
result = pipeline.run(data)
for subscriber in context.subscribers[name]:
try:
Send(subscriber.host, subscriber.port, result)
return result
class SubscribeOnPipeline(HandlerBase):
"""
Обработчик подписки на результаты работы пайплайна
"""
def __init__(self):
super().__init__("subscribe_on_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "host", "port"])
name = parameters["name"]
host = parameters["host"]
port = parameters["port"]
context.sub
return result
@@ -1,17 +0,0 @@
import json
from modules.handlers.base.handler_base import *
class CreatePipeline(HandlerBase):
def __init__(self):
super().__init__("create_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "pipes"])
name = parameters["name"]
pipes = parameters["pipes"]
context.pipeline_manager.create(name, pipes)
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
context.storage.do(query)
@@ -1,18 +0,0 @@
from modules.handlers.base.handler_base import HandlerBase
class GetPipelines(HandlerBase):
def __init__(self):
super().__init__("get_pipelines")
def do(self, context, parameters):
result = []
for name, pipeline in context.pipeline_manager.pipelines.items():
item = {
"name": pipeline.name,
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
}
result.append(item)
return result
-21
View File
@@ -1,21 +0,0 @@
from modules.pipelines.pipes.pipe_base import ToString
from modules.handlers.base.handler_base import HandlerBase
class GetPipes(HandlerBase):
def __init__(self):
super().__init__("get_pipes")
def do(self, context, parameters):
result = []
for pipe_name, pipe in context.pipe_manager.pipes.items():
item = {
"name": pipe.name,
"description": pipe.description,
"input_data_type": ToString(pipe.input_data_type),
"output_data_type": ToString(pipe.output_data_type)
}
result.append(item)
return result
@@ -1,16 +0,0 @@
from modules.handlers.base.handler_base import *
class RunPipeline(HandlerBase):
def __init__(self):
super().__init__("run_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "data"])
name = parameters["name"]
data = parameters["data"]
pipeline = context.pipeline_manager.get(name)
result = pipeline.run(data)
return result
+14 -3
View File
@@ -6,13 +6,24 @@ class LogLevel(Enum):
Debug = 2
def __int__(self):
return self.value
return self.value[0]
def __str__(self):
if self.value == LogLevel.Error:
return "Error"
elif self.value == LogLevel.Warning:
return "Warning"
elif self.value == LogLevel.Debug:
return "Debug"
else:
return "Unknown log level"
class Logger:
def __init__(self, max_log_level: LogLevel, callback):
self.max_log_level = max_log_level
self.callback = callback
def log(self, level, message):
if self.callback is not None and int(level) <= int(self.max_log_level):
def log(self, level: LogLevel, message):
# if self.callback is not None and int(level) <= int(self.max_log_level):
if self.callback:
self.callback(level, message)
+45
View File
@@ -0,0 +1,45 @@
import signal
import kafka
import kafka.errors
import multiprocessing.pool
class Config:
def __init__(self, topic, server, pool_size, pool_cache_limit):
self.topic = topic
self.server = server
self.pool_size = pool_size
self.pool_cache_limit = pool_cache_limit
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
def getPoolCacheSize(self):
return len(self._cache)
class Consumer:
def __init__(self, config: Config, callback):
self.callback = callback
self.consumer = kafka.KafkaConsumer(
config.topic,
bootstrap_servers=[config.server],
)
self.should_stop = False
self.pool_cache_limit = config.pool_cache_limit
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
signal.signal(signal.SIGTERM, self.stop)
def handlePoolCacheExcess(self):
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
pass
def run(self, context):
print("consumer is running")
while not self.should_stop:
for message in self.consumer:
if self.should_stop:
break
self.callback(context, message.value.decode())
def stop(self):
self.should_stop = True
+14
View File
@@ -0,0 +1,14 @@
import kafka
class Config:
def __init__(self, server):
self.server = server
class Producer:
def __init__(self, config):
self.producer = kafka.KafkaProducer(
bootstrap_servers=[config.server]
)
def send(self, topic: str, message: str):
self.producer.send(topic, message.encode())
+3
View File
@@ -20,4 +20,7 @@ class Pipeline:
for pipe in self.pipes:
data = pipe.do(data)
if data is None:
return None
return data
+16
View File
@@ -19,6 +19,22 @@ class ToLowerCase(PipeBase):
def do(self, data):
return data.lower()
class Strip(PipeBase):
"""
Удаляет пробелы в начале и конце строки
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("strip", DataType.String, DataType.String)
self.description = "removes any leading and trailing whitespaces"
def do(self, data: str):
return data.strip()
class Split(PipeBase):
"""
Разбивает строку на токены через разделитель
+10 -3
View File
@@ -1,8 +1,16 @@
import psycopg2 as pg
class ConnectionData:
def __init__(self):
self.host = None
self.port = None
self.database = None
self.user = None
self.password = None
class Storage:
def __init__(self, host, port, database, user, password):
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
def __init__(self, connection_data):
self.connection = pg.connect(host=connection_data.host, port=connection_data.port, dbname=connection_data.database, user=connection_data.user, password=connection_data.password)
def do(self, query, should_return=False):
transaction = self.connection.cursor()
@@ -12,4 +20,3 @@ class Storage:
if should_return:
return transaction.fetchall()
-6
View File
@@ -9,10 +9,6 @@ class Server:
self.callback = callback
self.context = None
self.server = None
self.logger = Logger(LogLevel.Error, None)
def setLogger(self, max_log_level, callback):
self.logger = Logger(max_log_level, callback)
async def run(self):
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
@@ -42,11 +38,9 @@ class Server:
request = request.decode()
# Вызываем коллбэк
response = self.callback(self.context, request)
response = struct.pack(">I", len(response)) + response.encode()
# Отправляем ответ
writer.write(response)
def stop(self):
@@ -0,0 +1,26 @@
class NgramInfo:
def __init__(self, id):
self.id = id
self.frequency = 1
class NgramsDictionary:
def __init__(self, ngram_size):
self.ngrams = {}
self.ngram_size = ngram_size
self.last_id = 0
def update(self, words):
for last_word_index in range(self.ngram_size, len(words) + 1):
ngram = ""
for current_word_index in range(0, self.ngram_size):
ngram += words[last_word_index - self.ngram_size + current_word_index] + " "
self.addNgram(ngram[:-1])
def addNgram(self, ngram):
if ngram in self.ngrams:
self.ngrams[ngram].frequency += 1
else:
word_info = NgramInfo(self.last_id)
self.last_id += 1
self.ngrams[ngram] = word_info
+20
View File
@@ -0,0 +1,20 @@
class Constants:
whitespace = r"\s*"
unknown_begin_part = r"<(\d{1,2})>"
month = r"([A-Za-z]{3})"
number = r"(\d+)"
time = r"(\d{2}:\d{2}:\d{2})"
word = r"(\S+)"
anything = r"(.*)"
class Pattern:
def __int__(self):
self.parts = []
def __str__(self):
result = ""
for part in self.parts:
result += part[0] + Constants.whitespace
return result
-25
View File
@@ -12,28 +12,3 @@
Так как модели могут потреблять много ресурсов, они не встраиваются напрямую в пайпы, а работают в отдельном процессе, с которым можно установить соединение (например, через сокет).
Таким образом, часть конвейера может выполняться там же, где был получен запрос на обработку, а часть -- на удалённой машине или в другом процессе.
## Использование
### Сервис Control
Для начала необходимо настроить хранилище, в которую будут сохраняться данные. В качестве хранилища используется PostgreSQL.
В директории sql располагается файл storage.sql. Необходимо:
1. Создать базу данных "log_analysis"
2. В созданной базе данных создать таблицу "pipelines"
После этого можно запускать сервис Control. При запуске необходимо указать параметры. Список этих параметров можно посмотреть в коде, либо запустить сервис без указания аргументов.
Сервис готов к работе после вывода надписи "server is running"
К сервису можно подключиться с помощью ConsoleClient (директория services/console_client). При запуске также нужно указать параметры.
#### Команды
##### Получение пайп
Возвращает список доступных пайп. Этот список формируется при запуске сервиса и в дальнейшем не изменяется.
##### Получение пайплайнов
Возвращает список доступных пайплайнов. Список может быть дополнен пользовательскими пайпланами.
##### Создание пайплайна
Создаёт пайплайн из указанных пайп. Пайпы должны соответствовать друг другу (то есть выход предыдущего пайпа должен соответстовать входу следующего)
##### Запуск пайплайна
Запускает пайплайн для обработки указанной строки. Возвращает результат обработки
-193
View File
@@ -1,193 +0,0 @@
import json
from modules.transports.client import Client
class Settings:
def __init__(self):
self.host = None
self.port = None
class CommandException(Exception):
def __init__(self, message):
super().__init__(message)
class Command:
def __init__(self, name):
"""
Конструктор
:param name: Название команды
"""
self.name = name
self.settings = None
def do(self, parameters):
"""
Вызов команды
:param parameters: Параметры
:return: Результат работы
"""
raise Exception("calling unimplemented method")
def doRequest(self, request: dict):
"""
Выполнение запроса
:param request: Тело запроса
:return: Ответ
"""
# Создаём клиентский сокет и подключаемся к серверу
client = Client()
client.connect(self.settings.host, self.settings.port)
# Отправляем запрос
client.send(json.dumps(request))
# Получаем ответ
response = client.receive()
# Закрываем соединение
client.disconnect()
# Парсим ответ
response = json.loads(response)
# Проверяем статус обработки запроса
status = response["status"]
if status != "ok":
message = response["message"]
raise CommandException(f"status: {status}, message: {message}")
return response["result"]
class CommandManager:
def __init__(self, settings):
self.settings = settings
self.commands = []
self.command_name_to_index = {}
def addCommand(self, command: Command):
command.settings = self.settings
self.command_name_to_index[command.name] = len(self.commands)
self.commands.append(command)
def hasCommand(self, name) -> bool:
if str.isnumeric(name):
index = int(name)
if index < 0 or index >= len(self.commands):
return False
return True
return name in self.command_name_to_index
def getCommand(self, name):
if not self.hasCommand(name):
raise Exception("there is no command with such name")
if str.isnumeric(name):
return self.commands[int(name)]
return self.commands[self.command_name_to_index[name]]
class GetPipes(Command):
"""
Получение списка пайп
"""
def __init__(self):
super().__init__("get_pipes")
def do(self, parameters):
request = {
"handler": self.name,
"parameters": ""
}
# Ответ -- это список пайп
response = self.doRequest(request)
if len(response) == 0:
print("there are no pipes")
return
for pipe in response[:-1]:
self.printPipe(pipe)
print()
self.printPipe(response[-1])
def printPipe(self, pipe):
print(pipe["name"] + ":")
print("- description: " + pipe["description"])
print("- input data type: " + pipe["input_data_type"])
print("- output data type: " + pipe["output_data_type"])
class GetPipelines(Command):
"""
Получение списка пайплайнов
"""
def __init__(self):
super().__init__("get_pipelines")
def do(self, parameters):
request = {
"handler": self.name,
"parameters": ""
}
# Ответ -- это список пайплайнов
response = self.doRequest(request)
if len(response) == 0:
print("there are no pipelines")
return
for pipeline in response:
print(pipeline["name"] + ": " + str(pipeline["pipes"]))
class RunPipeline(Command):
"""
Запуск пайплайна
"""
def __init__(self):
super().__init__("run_pipeline")
def do(self, parameters):
pipeline = input("pipeline: ")
data = input("data: ")
request = {
"handler": self.name,
"parameters": {
"name": pipeline,
"data": data
}
}
response = self.doRequest(request)
print("result: " + str(response))
class CreatePipeline(Command):
"""
Создание пайплайна
"""
def __init__(self):
super().__init__("create_pipeline")
def do(self, parameters):
name = input("name: ")
pipes = input("pipes (separated by spaces): ")
request = {
"handler": self.name,
"parameters": {
"name": name,
"pipes": pipes.split()
}
}
self.doRequest(request)
print("created successfully")
-70
View File
@@ -1,70 +0,0 @@
import argparse
from services.console_client.commands import *
def CreateCommandManager(settings):
manager = CommandManager(settings)
manager.addCommand(GetPipes())
manager.addCommand(GetPipelines())
manager.addCommand(CreatePipeline())
manager.addCommand(RunPipeline())
return manager
def PrintHelp(command_manager):
for index, command in enumerate(command_manager.commands):
print(str(index) + ". " + command.name)
def ParseArguments():
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
# Хост сервера
parser.add_argument("--host", type=str, required=True, help="server's host")
# Порт
parser.add_argument("--port", type=int, required=True, help="server's port")
arguments = parser.parse_args()
return arguments
def CreateSettings() -> Settings:
arguments = ParseArguments()
settings = Settings()
settings.host = arguments.host
settings.port = arguments.port
return settings
def main():
settings = CreateSettings()
command_manager = CreateCommandManager(settings)
parameters = {}
PrintHelp(command_manager)
while True:
command_name = str(input("> "))
if command_name in ["q", "quit", "exit"]:
break
if command_name in ["h", "help"]:
PrintHelp(command_manager)
continue
if not command_manager.hasCommand(command_name):
print("there is no command with such name")
continue
command = command_manager.getCommand(command_name)
try:
command.do(parameters)
except Exception as e:
print(f"error: {str(e)}")
print()
if __name__ == "__main__":
main()
-19
View File
@@ -1,19 +0,0 @@
from modules.pipelines.pipe_manager import PipeManager
from modules.pipelines.pipeline_manager import PipelineManager
from modules.handlers.handler_manager import HandlerManager
from modules.storages.storage import Storage
class Context:
"""
Контекст управляющего сервиса
"""
def __init__(self):
"""
Конструктор
"""
self.pipe_manager = None
self.pipeline_manager = None
self.handler_manager = None
self.storage = None
-130
View File
@@ -1,130 +0,0 @@
import argparse
import asyncio
import json
from modules.logging.logger import *
from modules.storages.storage import Storage
from modules.transports.server import Server
from services.control.context import *
from services.control.create_context import *
from services.control.response import *
def Handle(context, request):
"""
Обработка запроса
:param context: Контекст сервиса
:param request: Запрос
:return: Ответ
"""
# Парсим запрос
try:
request = json.loads(request)
except Exception as e:
return ErrorResponse("invalid request", "failed to parse request")
# Проверяем наличие нужных ключей
for key in ["handler", "parameters"]:
if key not in request:
return ErrorResponse("invalid request", f"key '{key}' is required")
# Проверяем наличие обработчика с указанным именем
handler_name = request["handler"]
if not context.handler_manager.contains(handler_name):
return ErrorResponse("not found", f"there is no handler with name {handler_name}")
# Достаём нужного обработчика
handler = context.handler_manager.get(handler_name)
# Запускаем обработку
try:
result = handler.do(context, request["parameters"])
return SuccessResponse(result)
except Exception as e:
return ErrorResponse("handling failed", str(e))
def ParseArguments():
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
# Данные сервера
parser.add_argument("--host", type=str, required=True, help="server's host")
parser.add_argument("--port", type=int, required=True, help="server's port")
# Данные для подключения к базе данных
parser.add_argument("--psql-host", type=str, required=True, help="postgresql's host")
parser.add_argument("--psql-port", type=str, required=True, help="postgresql's port")
parser.add_argument("--psql-database", type=str, required=True, help="postgresql's database")
parser.add_argument("--psql-user", type=str, required=True, help="postgresql's user")
parser.add_argument("--psql-password", type=str, required=True, help="postgresql's password")
arguments = parser.parse_args()
return arguments
class PostgreSqlSettings:
def __init__(self):
self.host = None
self.port = None
self.database = None
self.user = None
self.password = None
class Settings:
def __init__(self):
self.host = None
self.port = None
self.postgresql_settings = None
def CreateSettings():
arguments = ParseArguments()
postgresql_settings = PostgreSqlSettings()
postgresql_settings.host = arguments.psql_host
postgresql_settings.port = arguments.psql_port
postgresql_settings.database = arguments.psql_database
postgresql_settings.user = arguments.psql_user
postgresql_settings.password = arguments.psql_password
settings = Settings()
settings.host = arguments.host
settings.port = arguments.port
settings.postgresql_settings = postgresql_settings
return settings
def LogCallback(level, message):
print(f"level: {level}, message: {message}")
def main():
settings = CreateSettings()
server = Server(settings.host, settings.port, Handle)
server.setLogger(LogLevel.Warning, LogCallback)
# Создаём контекст
context = CreateContext()
# Подключаемся к хранилищу
context.storage = Storage(
settings.postgresql_settings.host,
settings.postgresql_settings.port,
settings.postgresql_settings.database,
settings.postgresql_settings.user,
settings.postgresql_settings.password
)
# Заполняем пайплайны
query = "select name, pipes from pipelines;"
pipelines = context.storage.do(query, should_return=True)
for name, pipes in pipelines:
context.pipeline_manager.create(name, json.loads(pipes))
# Указываем контекст в сервере
server.context = context
# Запускаем сервер
asyncio.run(server.run())
if __name__ == "__main__":
main()
-62
View File
@@ -1,62 +0,0 @@
from modules.pipelines.pipe_manager import PipeManager
from modules.pipelines.pipeline_manager import PipelineManager
from modules.pipelines.pipes.preprocessing import *
from modules.handlers.handler_manager import HandlerManager
from modules.handlers.pipelines.create_pipeline import CreatePipeline
from modules.handlers.pipelines.get_pipelines import GetPipelines
from modules.handlers.pipelines.get_pipes import GetPipes
from modules.handlers.pipelines.run_pipeline import RunPipeline
from services.control.context import Context
def CreatePipeManager() -> PipeManager:
"""
Создание заполненного менеджера пайп
:return: Менеджер пайп
"""
manager = PipeManager()
manager.add(RemovePunctuationMarks())
manager.add(RemoveStopWords())
manager.add(Stem())
manager.add(ToLowerCase())
manager.add(SplitBySpace())
return manager
def CreatePipelineManager(pipe_manager) -> PipelineManager:
"""
Создание заполненного менеджера пайплайнов
:param pipe_manager: Менеджер пайп
:return: Менеджер пайплайнов
"""
manager = PipelineManager(pipe_manager)
return manager
def CreateHandlerManager() -> HandlerManager:
"""
Создание менеджера хэндлеров
:return: Менеджер хэндлеров
"""
manager = HandlerManager()
manager.add(CreatePipeline())
manager.add(GetPipelines())
manager.add(GetPipes())
manager.add(RunPipeline())
return manager
def CreateContext() -> Context:
"""
Создание заполненного контекста
:return: Контекст
"""
context = Context()
context.pipe_manager = CreatePipeManager()
context.pipeline_manager = CreatePipelineManager(context.pipe_manager)
context.handler_manager = CreateHandlerManager()
return context
-26
View File
@@ -1,26 +0,0 @@
import json
def SuccessResponse(result: str) -> str:
"""
Создание успешного ответа
:param result: Тело ответа
:return: Ответ
"""
return json.dumps({
"status": "ok",
"result": result
})
def ErrorResponse(status: str, message: str) -> str:
"""
Создание ответа с ошибкой
:param status: Код ошибки
:param message: Сообщение об ошибке
:return:
"""
return json.dumps({
"status": status,
"message": message
})
+41
View File
@@ -0,0 +1,41 @@
import re
from modules.logging.logger import LogLevel
from modules.utils.pattern import Constants, Pattern
from services.dispatchers.dispatcher_base import DispatcherBase
def ConsumeCallback(context, message):
match = re.search(str(context.pattern), message)
if match is None:
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
return
groups = match.groups()
topic = groups[5]
message = groups[6]
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
context.producer.send(topic, message)
class AllDispatcher(DispatcherBase):
def __init__(self):
# Паттерн: <number>month number hh:mm:ss machine-name process-name: message
pattern = Pattern()
pattern.parts = [
(Constants.unknown_begin_part, "unknown begin part"), # 0
(Constants.month, "month"), # 1
(Constants.number, "unknown number"), # 2
(Constants.time, "time"), # 3
(Constants.word, "machine name"), # 4
(Constants.word + ":", "process name"), # 5
(Constants.anything, "message") # 6
]
super().__init__(topic="all", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for all logs")
def main():
dispatcher = AllDispatcher()
dispatcher.run()
if __name__ == "__main__":
main()
@@ -0,0 +1,43 @@
import re
from modules.logging.logger import LogLevel
from modules.utils.pattern import Constants, Pattern
from services.dispatchers.dispatcher_base import DispatcherBase
def ConsumeCallback(context, message):
match = re.search(str(context.pattern), message)
if match is None:
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
return
groups = match.groups()
topic = groups[3]
message = groups[4]
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
context.producer.send(topic, message)
class CephOsdDispatcher(DispatcherBase):
def __init__(self):
# Паттерн: yyyy-mm-ddThh:mm:ss+timezone pid number process-name: message
datetime = Constants.number + "-" + Constants.number + "-" + Constants.number + "T" + Constants.time + "\." + Constants.number + "\+" + Constants.number
datetime = datetime.replace("(", "")
datetime = datetime.replace(")", "")
pattern = Pattern()
pattern.parts = [
("(" + datetime + ")", "Date and time with timezone"), # 0
(Constants.word, "pid"), # 1
(Constants.number, "unknown number"), # 2
(Constants.word + ":", "process name"), # 3
(Constants.anything, "message") # 4
]
super().__init__(topic="ceph-osd", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for ceph-osd logs")
def main():
dispatcher = CephOsdDispatcher()
dispatcher.run()
if __name__ == "__main__":
main()
+51
View File
@@ -0,0 +1,51 @@
import argparse
from datetime import datetime
from modules.logging.logger import LogLevel, Logger
from modules.message_broker.consumer import Config as ConsumerConfig, Consumer
from modules.message_broker.producer import Config as ProducerConfig, Producer
from modules.utils.pattern import Pattern
class Context:
def __init__(self):
self.logger = None
self.pattern = None
self.producer = None
def LogCallback(level, message):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{str(level)}] {now}: {message}")
def ParseArguments(description):
parser = argparse.ArgumentParser(description=description)
parser.add_argument("--server", type=str, required=True, help="Kafka server")
parser.add_argument("--pool-size", type=int, required=True, help="Pool size")
parser.add_argument("--pool-cache-limit", type=int, required=True, help="Pool cache limit")
parser.add_argument("--max-log-level", type=int, required=False, help="Max log level. Error -- 0 (default), Warning -- 1, Debug -- 2")
arguments = parser.parse_args()
return arguments
class DispatcherBase:
def __init__(self, topic: str, pattern: Pattern, consume_callback, description: str):
# Парсим аргументы
arguments = ParseArguments(description)
# Создаём контекст
self.context = Context()
self.context.pattern = pattern
# Настраиваем логирование
max_log_level = LogLevel.Debug # todo log level
self.context.logger = Logger(max_log_level=max_log_level, callback=LogCallback)
# Создаём продюсера
producer_config = ProducerConfig(arguments.server)
self.context.producer = Producer(producer_config)
# Создаём консюмера
consumer_config = ConsumerConfig(topic, arguments.server, arguments.pool_size, arguments.pool_cache_limit)
self.consumer = Consumer(consumer_config, consume_callback)
def run(self):
self.consumer.run(self.context)
+4
View File
@@ -0,0 +1,4 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+138
View File
@@ -0,0 +1,138 @@
import argparse
import json
import re
from modules.message_broker.consumer import Config, Consumer
from modules.storages.storage import ConnectionData, Storage
from modules.utils.pattern import Constants, Pattern
from services.log_analyzer.model import LogAnalyzer
class Context:
def __init__(self):
self.kafka_server = None
self.topic = None
self.model = None
self.storage = None
self.message_pattern = None
self.enable_logging = False
def CheckRequiredKeys(config):
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
is_ok = True
for key in keys:
if key not in config:
print(f"config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
storage_config = config["storage"]
storage_keys = ["host", "port", "database", "user", "password"]
for key in storage_keys:
if key not in storage_config:
print(f"storage config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
def ParseConfig() -> Context:
parser = argparse.ArgumentParser("Log analyzer")
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
arguments = parser.parse_args()
with open(arguments.config, "r") as file:
config_data = file.read()
config = json.loads(config_data)
CheckRequiredKeys(config)
context = Context()
context.kafka_server = config["kafka_server"]
context.topic = config["topic"]
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
context.model.fit(config["model"])
storage_config = config["storage"]
connection_data = ConnectionData()
connection_data.host = storage_config["host"]
connection_data.port = storage_config["port"]
connection_data.database = storage_config["database"]
connection_data.user = storage_config["user"]
connection_data.password = storage_config["password"]
context.storage = Storage(connection_data)
context.enable_logging = arguments.enable_logging
pattern = Pattern()
pattern.parts = [
(Constants.unknown_begin_part, "unknown begin part"), # 0
(Constants.month, "month"), # 1
(Constants.number, "unknown number"), # 2
(Constants.time, "time"), # 3
(Constants.word, "machine name"), # 4
(Constants.word + ":", "process name"), # 5
(Constants.anything, "message") # 6
]
context.message_pattern = re.compile(str(pattern))
return context
def ClearMessage(context: Context, message: str) -> tuple:
match = re.search(context.message_pattern, message)
if match is None:
if context.enable_logging:
print(f"Nothing is matched for log '{message}'")
return "", message
groups = match.groups()
topic = groups[5]
message = groups[6]
return topic, message
def UpdateStatisticsQuery(name: str):
return f"update statistics set value = value + 1 where name = '{name}';"
def SaveAnomalyLogQuery(log: str):
return f"insert into anomaly_logs (data) values('{log}')"
def Callback(context: Context, message: str):
if context.enable_logging:
print(f"Processing message '{message}'", message)
topic, message = ClearMessage(context, message)
result = context.model.predict(program=topic, message=message)[0]
result_as_str = "anomaly" if result == -1 else "not anomaly"
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
context.storage.do(query)
if result == -1:
query = SaveAnomalyLogQuery(message)
context.storage.do(query)
if context.enable_logging:
print("log is " + result_as_str)
def main():
# Создаём контекст
context = ParseConfig()
# Создаём потребителя
config = Config(context.topic, context.kafka_server, 16, 16)
consumer = Consumer(config, Callback)
# Запускаем обработку
consumer.run(context)
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
from transformers import AutoTokenizer, AutoModel, AutoConfig
import torch
import torch.nn.functional as F
import pickle
import numpy
import pandas as pd
class LogAnalyzer:
'''Сlass of an expert system module that analyzes diagnostic information
about the operation of the input-output subsystem.
'''
def __init__(self, path_model_embedding: str, path_tokenizer: str):
'''Constructor of model for analyse.
:param path_model_embedding: String of path to directory with model
for build embeddings of message
:param path_tokenizer: String of path to directory with tokenizer
for tokenize message
'''
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
self.model = None
def predict(self, program: str, message: str) -> numpy.ndarray:
'''Function predict whether an log is anomaly.
:param message: string of message
'''
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
'''Function applies mean pooling on the embedding model to get
a single vector representation.
:param model_output: Tuple containing different model outputs.
The first element contains the token embeddings.
:param attention_mask: Tensor that identifies the tokens
to which attention should be paid (value of 1)
versus those which should not (value of 0).
:return: Tensor representing the pooled output of the input tokens.
'''
token_embeddings = model_output[0]
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
token_embeddings.size()).float()
return torch.sum(
token_embeddings * input_mask_expanded, 1) / torch.clamp(
input_mask_expanded.sum(1), min=1e-9)
def embed(message: str, tokenizer, model) -> torch.Tensor:
'''Function generates embeddings for a given message.
:param message: String of message for which embeddings
are to be generated.
:param tokenizer: The tokenizer to be used for tokenizing
the message.
:param model: The model to be used for generating embeddings.
:return: Tensor of normalized embeddings of the input message.
'''
encoded_input = tokenizer(
message, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
model_output = model(**encoded_input)
message_embeddings_mean = mean_pooling(
model_output, encoded_input['attention_mask'])
message_embeddings = F.normalize(
message_embeddings_mean, p=2, dim=1)
return message_embeddings
if self.model is None:
raise ValueError("Model is not fitted. Make '.fit' method first")
return None
message_embeddings = embed(
message, self.tokenizer, self.model_embedding)
try:
prediction = self.model[program].predict(
message_embeddings.numpy())
except KeyError:
prediction = self.model['unknown'].predict(
message_embeddings.numpy())
return prediction
def fit(self, path_model):
'''Function predict whether an log is anomaly.
:param path_model: String of path to saved model
'''
self.model = pickle.load(open(path_model, 'rb'))
+33
View File
@@ -0,0 +1,33 @@
import re
from modules.pipelines.pipes.preprocessing import DataType, PipeBase
class RemoveByRegex(PipeBase):
def __init__(self, name, pattern):
super().__init__(name, DataType.String, DataType.String)
self.pattern = re.compile(pattern)
def do(self, data):
return self.pattern.sub("", data)
class RemoveOriginalLogTime(RemoveByRegex):
def __init__(self):
super().__init__("remove_original_log_time", "\(Original Log Time .*\)")
class RemoveSourceCode(RemoveByRegex):
def __init__(self):
super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]")
class StopRunningIfContains(PipeBase):
def __init__(self, items: list[str]):
super().__init__("stop_running_if_contains", DataType.String, DataType.String)
self.items = items
def do(self, data: str):
for item in self.items:
if data.find(item) != -1:
return None
return data
@@ -0,0 +1,22 @@
from modules.pipelines.pipeline import Pipeline
from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace
from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains
def main():
pipeline = Pipeline("prepare")
pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"]))
pipeline.add(RemoveOriginalLogTime())
pipeline.add(RemoveSourceCode())
pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]"))
pipeline.add(Strip())
with open("rocksdb_logs.txt", "r") as input:
with open("filtered_rocksdb_logs.txt", "w") as output:
for line in input.readlines():
result = pipeline.run(line)
if result is not None:
output.write(result + "\n")
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
import argparse
from kafka import KafkaConsumer
parser = argparse.ArgumentParser()
parser.add_argument("--topic", type=str, required=True)
arguments = parser.parse_args()
consumer = KafkaConsumer(bootstrap_servers=["localhost:29092"])
consumer.subscribe(arguments.topic)
for message in consumer:
print(message.value)
+12
View File
@@ -0,0 +1,12 @@
import argparse
from kafka import KafkaProducer
parser = argparse.ArgumentParser()
parser.add_argument("--topic", type=str, required=True)
parser.add_argument("--message", type=str, required=True)
arguments = parser.parse_args()
producer = KafkaProducer(bootstrap_servers=["localhost:29092"])
future = producer.send(arguments.topic, arguments.message.encode())
future.get(timeout=3)
-7
View File
@@ -1,7 +0,0 @@
create database log_analysis;
create table pipelines(
id serial,
name text,
pipes text
);
+15
View File
@@ -0,0 +1,15 @@
create database log_analysis;
create table statistics(
name varchar(64),
value int,
primary key (name)
);
insert into statistics (name, value) values ('total', 0), ('anomaly', 0), ('not anomaly', 0), ('undefined', 0);
create table anomaly_logs(
id serial,
data text,
primary key (id)
);