Большое обновление всего проекта
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
version: '2'
|
||||
services:
|
||||
zookeeper:
|
||||
image: confluentinc/cp-zookeeper:latest
|
||||
environment:
|
||||
ZOOKEEPER_CLIENT_PORT: 2181
|
||||
ZOOKEEPER_TICK_TIME: 2000
|
||||
ports:
|
||||
- 22181:2181
|
||||
|
||||
kafka:
|
||||
image: confluentinc/cp-kafka:latest
|
||||
depends_on:
|
||||
- zookeeper
|
||||
ports:
|
||||
- 29092:29092
|
||||
environment:
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
||||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
class Subscriber:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
|
||||
class Context:
|
||||
"""
|
||||
Контекст обработчиков
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
self.pipe_manager = None
|
||||
self.pipeline_manager = None
|
||||
self.subscribers = {}
|
||||
@@ -1,3 +1,5 @@
|
||||
from modules.handlers.context import Context
|
||||
|
||||
def CheckParameters(parameters, required_keys):
|
||||
for key in required_keys:
|
||||
if key not in parameters:
|
||||
@@ -7,5 +9,5 @@ class HandlerBase:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def do(self, context, parameters):
|
||||
def do(self, context: Context, parameters: dict):
|
||||
pass
|
||||
@@ -0,0 +1,109 @@
|
||||
import json
|
||||
|
||||
from modules.pipelines.pipes.pipe_base import ToString
|
||||
from modules.handlers.handler_base import *
|
||||
|
||||
class CreatePipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик создания пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "pipes"])
|
||||
|
||||
name = parameters["name"]
|
||||
pipes = parameters["pipes"]
|
||||
|
||||
context.pipeline_manager.create(name, pipes)
|
||||
|
||||
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
|
||||
context.storage.do(query)
|
||||
|
||||
class GetPipelines(HandlerBase):
|
||||
"""
|
||||
Обработчик получения списка пайплайнов
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
result = []
|
||||
|
||||
for name, pipeline in context.pipeline_manager.pipelines.items():
|
||||
item = {
|
||||
"name": pipeline.name,
|
||||
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
|
||||
class GetPipes(HandlerBase):
|
||||
"""
|
||||
Обработчик получения списка пайп
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
result = []
|
||||
|
||||
for pipe_name, pipe in context.pipe_manager.pipes.items():
|
||||
item = {
|
||||
"name": pipe.name,
|
||||
"description": pipe.description,
|
||||
"input_data_type": ToString(pipe.input_data_type),
|
||||
"output_data_type": ToString(pipe.output_data_type)
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
|
||||
class RunPipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик запуска пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "data"])
|
||||
|
||||
name = parameters["name"]
|
||||
data = parameters["data"]
|
||||
|
||||
pipeline = context.pipeline_manager.get(name)
|
||||
result = pipeline.run(data)
|
||||
|
||||
for subscriber in context.subscribers[name]:
|
||||
try:
|
||||
Send(subscriber.host, subscriber.port, result)
|
||||
|
||||
return result
|
||||
|
||||
class SubscribeOnPipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик подписки на результаты работы пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("subscribe_on_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "host", "port"])
|
||||
|
||||
name = parameters["name"]
|
||||
host = parameters["host"]
|
||||
port = parameters["port"]
|
||||
|
||||
context.sub
|
||||
|
||||
return result
|
||||
@@ -1,17 +0,0 @@
|
||||
import json
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class CreatePipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "pipes"])
|
||||
|
||||
name = parameters["name"]
|
||||
pipes = parameters["pipes"]
|
||||
|
||||
context.pipeline_manager.create(name, pipes)
|
||||
|
||||
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
|
||||
context.storage.do(query)
|
||||
@@ -1,18 +0,0 @@
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipelines(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for name, pipeline in context.pipeline_manager.pipelines.items():
|
||||
item = {
|
||||
"name": pipeline.name,
|
||||
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -1,21 +0,0 @@
|
||||
from modules.pipelines.pipes.pipe_base import ToString
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipes(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for pipe_name, pipe in context.pipe_manager.pipes.items():
|
||||
item = {
|
||||
"name": pipe.name,
|
||||
"description": pipe.description,
|
||||
"input_data_type": ToString(pipe.input_data_type),
|
||||
"output_data_type": ToString(pipe.output_data_type)
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -1,16 +0,0 @@
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class RunPipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "data"])
|
||||
|
||||
name = parameters["name"]
|
||||
data = parameters["data"]
|
||||
|
||||
pipeline = context.pipeline_manager.get(name)
|
||||
result = pipeline.run(data)
|
||||
|
||||
return result
|
||||
@@ -6,13 +6,24 @@ class LogLevel(Enum):
|
||||
Debug = 2
|
||||
|
||||
def __int__(self):
|
||||
return self.value
|
||||
return self.value[0]
|
||||
|
||||
def __str__(self):
|
||||
if self.value == LogLevel.Error:
|
||||
return "Error"
|
||||
elif self.value == LogLevel.Warning:
|
||||
return "Warning"
|
||||
elif self.value == LogLevel.Debug:
|
||||
return "Debug"
|
||||
else:
|
||||
return "Unknown log level"
|
||||
|
||||
class Logger:
|
||||
def __init__(self, max_log_level: LogLevel, callback):
|
||||
self.max_log_level = max_log_level
|
||||
self.callback = callback
|
||||
|
||||
def log(self, level, message):
|
||||
if self.callback is not None and int(level) <= int(self.max_log_level):
|
||||
def log(self, level: LogLevel, message):
|
||||
# if self.callback is not None and int(level) <= int(self.max_log_level):
|
||||
if self.callback:
|
||||
self.callback(level, message)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import signal
|
||||
import kafka
|
||||
import kafka.errors
|
||||
import multiprocessing.pool
|
||||
|
||||
class Config:
|
||||
def __init__(self, topic, server, pool_size, pool_cache_limit):
|
||||
self.topic = topic
|
||||
self.server = server
|
||||
self.pool_size = pool_size
|
||||
self.pool_cache_limit = pool_cache_limit
|
||||
|
||||
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
|
||||
def getPoolCacheSize(self):
|
||||
return len(self._cache)
|
||||
|
||||
class Consumer:
|
||||
def __init__(self, config: Config, callback):
|
||||
self.callback = callback
|
||||
self.consumer = kafka.KafkaConsumer(
|
||||
config.topic,
|
||||
bootstrap_servers=[config.server],
|
||||
)
|
||||
|
||||
self.should_stop = False
|
||||
self.pool_cache_limit = config.pool_cache_limit
|
||||
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
|
||||
signal.signal(signal.SIGTERM, self.stop)
|
||||
|
||||
def handlePoolCacheExcess(self):
|
||||
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
|
||||
pass
|
||||
|
||||
def run(self, context):
|
||||
print("consumer is running")
|
||||
|
||||
while not self.should_stop:
|
||||
for message in self.consumer:
|
||||
if self.should_stop:
|
||||
break
|
||||
|
||||
self.callback(context, message.value.decode())
|
||||
|
||||
def stop(self):
|
||||
self.should_stop = True
|
||||
@@ -0,0 +1,14 @@
|
||||
import kafka
|
||||
|
||||
class Config:
|
||||
def __init__(self, server):
|
||||
self.server = server
|
||||
|
||||
class Producer:
|
||||
def __init__(self, config):
|
||||
self.producer = kafka.KafkaProducer(
|
||||
bootstrap_servers=[config.server]
|
||||
)
|
||||
|
||||
def send(self, topic: str, message: str):
|
||||
self.producer.send(topic, message.encode())
|
||||
@@ -20,4 +20,7 @@ class Pipeline:
|
||||
for pipe in self.pipes:
|
||||
data = pipe.do(data)
|
||||
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
@@ -19,6 +19,22 @@ class ToLowerCase(PipeBase):
|
||||
def do(self, data):
|
||||
return data.lower()
|
||||
|
||||
class Strip(PipeBase):
|
||||
"""
|
||||
Удаляет пробелы в начале и конце строки
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("strip", DataType.String, DataType.String)
|
||||
self.description = "removes any leading and trailing whitespaces"
|
||||
|
||||
def do(self, data: str):
|
||||
return data.strip()
|
||||
|
||||
class Split(PipeBase):
|
||||
"""
|
||||
Разбивает строку на токены через разделитель
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
import psycopg2 as pg
|
||||
|
||||
class ConnectionData:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.database = None
|
||||
self.user = None
|
||||
self.password = None
|
||||
|
||||
class Storage:
|
||||
def __init__(self, host, port, database, user, password):
|
||||
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
|
||||
def __init__(self, connection_data):
|
||||
self.connection = pg.connect(host=connection_data.host, port=connection_data.port, dbname=connection_data.database, user=connection_data.user, password=connection_data.password)
|
||||
|
||||
def do(self, query, should_return=False):
|
||||
transaction = self.connection.cursor()
|
||||
@@ -12,4 +20,3 @@ class Storage:
|
||||
|
||||
if should_return:
|
||||
return transaction.fetchall()
|
||||
|
||||
|
||||
@@ -9,10 +9,6 @@ class Server:
|
||||
self.callback = callback
|
||||
self.context = None
|
||||
self.server = None
|
||||
self.logger = Logger(LogLevel.Error, None)
|
||||
|
||||
def setLogger(self, max_log_level, callback):
|
||||
self.logger = Logger(max_log_level, callback)
|
||||
|
||||
async def run(self):
|
||||
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
|
||||
@@ -42,11 +38,9 @@ class Server:
|
||||
|
||||
request = request.decode()
|
||||
|
||||
# Вызываем коллбэк
|
||||
response = self.callback(self.context, request)
|
||||
response = struct.pack(">I", len(response)) + response.encode()
|
||||
|
||||
# Отправляем ответ
|
||||
writer.write(response)
|
||||
|
||||
def stop(self):
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
class NgramInfo:
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.frequency = 1
|
||||
|
||||
class NgramsDictionary:
|
||||
def __init__(self, ngram_size):
|
||||
self.ngrams = {}
|
||||
self.ngram_size = ngram_size
|
||||
self.last_id = 0
|
||||
|
||||
def update(self, words):
|
||||
for last_word_index in range(self.ngram_size, len(words) + 1):
|
||||
ngram = ""
|
||||
for current_word_index in range(0, self.ngram_size):
|
||||
ngram += words[last_word_index - self.ngram_size + current_word_index] + " "
|
||||
|
||||
self.addNgram(ngram[:-1])
|
||||
|
||||
def addNgram(self, ngram):
|
||||
if ngram in self.ngrams:
|
||||
self.ngrams[ngram].frequency += 1
|
||||
else:
|
||||
word_info = NgramInfo(self.last_id)
|
||||
self.last_id += 1
|
||||
self.ngrams[ngram] = word_info
|
||||
@@ -0,0 +1,20 @@
|
||||
class Constants:
|
||||
whitespace = r"\s*"
|
||||
unknown_begin_part = r"<(\d{1,2})>"
|
||||
month = r"([A-Za-z]{3})"
|
||||
number = r"(\d+)"
|
||||
time = r"(\d{2}:\d{2}:\d{2})"
|
||||
word = r"(\S+)"
|
||||
anything = r"(.*)"
|
||||
|
||||
class Pattern:
|
||||
def __int__(self):
|
||||
self.parts = []
|
||||
|
||||
def __str__(self):
|
||||
result = ""
|
||||
|
||||
for part in self.parts:
|
||||
result += part[0] + Constants.whitespace
|
||||
|
||||
return result
|
||||
@@ -12,28 +12,3 @@
|
||||
|
||||
Так как модели могут потреблять много ресурсов, они не встраиваются напрямую в пайпы, а работают в отдельном процессе, с которым можно установить соединение (например, через сокет).
|
||||
Таким образом, часть конвейера может выполняться там же, где был получен запрос на обработку, а часть -- на удалённой машине или в другом процессе.
|
||||
|
||||
## Использование
|
||||
### Сервис Control
|
||||
Для начала необходимо настроить хранилище, в которую будут сохраняться данные. В качестве хранилища используется PostgreSQL.
|
||||
В директории sql располагается файл storage.sql. Необходимо:
|
||||
1. Создать базу данных "log_analysis"
|
||||
2. В созданной базе данных создать таблицу "pipelines"
|
||||
|
||||
После этого можно запускать сервис Control. При запуске необходимо указать параметры. Список этих параметров можно посмотреть в коде, либо запустить сервис без указания аргументов.
|
||||
Сервис готов к работе после вывода надписи "server is running"
|
||||
|
||||
К сервису можно подключиться с помощью ConsoleClient (директория services/console_client). При запуске также нужно указать параметры.
|
||||
|
||||
#### Команды
|
||||
##### Получение пайп
|
||||
Возвращает список доступных пайп. Этот список формируется при запуске сервиса и в дальнейшем не изменяется.
|
||||
|
||||
##### Получение пайплайнов
|
||||
Возвращает список доступных пайплайнов. Список может быть дополнен пользовательскими пайпланами.
|
||||
|
||||
##### Создание пайплайна
|
||||
Создаёт пайплайн из указанных пайп. Пайпы должны соответствовать друг другу (то есть выход предыдущего пайпа должен соответстовать входу следующего)
|
||||
|
||||
##### Запуск пайплайна
|
||||
Запускает пайплайн для обработки указанной строки. Возвращает результат обработки
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
import json
|
||||
|
||||
from modules.transports.client import Client
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
|
||||
class CommandException(Exception):
|
||||
def __init__(self, message):
|
||||
super().__init__(message)
|
||||
|
||||
class Command:
|
||||
def __init__(self, name):
|
||||
"""
|
||||
Конструктор
|
||||
:param name: Название команды
|
||||
"""
|
||||
self.name = name
|
||||
self.settings = None
|
||||
|
||||
def do(self, parameters):
|
||||
"""
|
||||
Вызов команды
|
||||
:param parameters: Параметры
|
||||
:return: Результат работы
|
||||
"""
|
||||
raise Exception("calling unimplemented method")
|
||||
|
||||
def doRequest(self, request: dict):
|
||||
"""
|
||||
Выполнение запроса
|
||||
:param request: Тело запроса
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
# Создаём клиентский сокет и подключаемся к серверу
|
||||
client = Client()
|
||||
client.connect(self.settings.host, self.settings.port)
|
||||
|
||||
# Отправляем запрос
|
||||
client.send(json.dumps(request))
|
||||
|
||||
# Получаем ответ
|
||||
response = client.receive()
|
||||
|
||||
# Закрываем соединение
|
||||
client.disconnect()
|
||||
|
||||
# Парсим ответ
|
||||
response = json.loads(response)
|
||||
|
||||
# Проверяем статус обработки запроса
|
||||
status = response["status"]
|
||||
if status != "ok":
|
||||
message = response["message"]
|
||||
raise CommandException(f"status: {status}, message: {message}")
|
||||
|
||||
return response["result"]
|
||||
|
||||
class CommandManager:
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.commands = []
|
||||
self.command_name_to_index = {}
|
||||
|
||||
def addCommand(self, command: Command):
|
||||
command.settings = self.settings
|
||||
self.command_name_to_index[command.name] = len(self.commands)
|
||||
self.commands.append(command)
|
||||
|
||||
def hasCommand(self, name) -> bool:
|
||||
if str.isnumeric(name):
|
||||
index = int(name)
|
||||
if index < 0 or index >= len(self.commands):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
return name in self.command_name_to_index
|
||||
|
||||
def getCommand(self, name):
|
||||
if not self.hasCommand(name):
|
||||
raise Exception("there is no command with such name")
|
||||
|
||||
if str.isnumeric(name):
|
||||
return self.commands[int(name)]
|
||||
|
||||
return self.commands[self.command_name_to_index[name]]
|
||||
|
||||
class GetPipes(Command):
|
||||
"""
|
||||
Получение списка пайп
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, parameters):
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": ""
|
||||
}
|
||||
|
||||
# Ответ -- это список пайп
|
||||
response = self.doRequest(request)
|
||||
|
||||
if len(response) == 0:
|
||||
print("there are no pipes")
|
||||
return
|
||||
|
||||
for pipe in response[:-1]:
|
||||
self.printPipe(pipe)
|
||||
print()
|
||||
|
||||
self.printPipe(response[-1])
|
||||
|
||||
def printPipe(self, pipe):
|
||||
print(pipe["name"] + ":")
|
||||
print("- description: " + pipe["description"])
|
||||
print("- input data type: " + pipe["input_data_type"])
|
||||
print("- output data type: " + pipe["output_data_type"])
|
||||
|
||||
class GetPipelines(Command):
|
||||
"""
|
||||
Получение списка пайплайнов
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, parameters):
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": ""
|
||||
}
|
||||
|
||||
# Ответ -- это список пайплайнов
|
||||
response = self.doRequest(request)
|
||||
|
||||
if len(response) == 0:
|
||||
print("there are no pipelines")
|
||||
return
|
||||
|
||||
for pipeline in response:
|
||||
print(pipeline["name"] + ": " + str(pipeline["pipes"]))
|
||||
|
||||
class RunPipeline(Command):
|
||||
"""
|
||||
Запуск пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, parameters):
|
||||
pipeline = input("pipeline: ")
|
||||
data = input("data: ")
|
||||
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": {
|
||||
"name": pipeline,
|
||||
"data": data
|
||||
}
|
||||
}
|
||||
|
||||
response = self.doRequest(request)
|
||||
print("result: " + str(response))
|
||||
|
||||
class CreatePipeline(Command):
|
||||
"""
|
||||
Создание пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, parameters):
|
||||
name = input("name: ")
|
||||
pipes = input("pipes (separated by spaces): ")
|
||||
|
||||
request = {
|
||||
"handler": self.name,
|
||||
"parameters": {
|
||||
"name": name,
|
||||
"pipes": pipes.split()
|
||||
}
|
||||
}
|
||||
|
||||
self.doRequest(request)
|
||||
print("created successfully")
|
||||
@@ -1,70 +0,0 @@
|
||||
import argparse
|
||||
|
||||
from services.console_client.commands import *
|
||||
|
||||
def CreateCommandManager(settings):
|
||||
manager = CommandManager(settings)
|
||||
manager.addCommand(GetPipes())
|
||||
manager.addCommand(GetPipelines())
|
||||
manager.addCommand(CreatePipeline())
|
||||
manager.addCommand(RunPipeline())
|
||||
|
||||
return manager
|
||||
|
||||
def PrintHelp(command_manager):
|
||||
for index, command in enumerate(command_manager.commands):
|
||||
print(str(index) + ". " + command.name)
|
||||
|
||||
def ParseArguments():
|
||||
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
|
||||
|
||||
# Хост сервера
|
||||
parser.add_argument("--host", type=str, required=True, help="server's host")
|
||||
|
||||
# Порт
|
||||
parser.add_argument("--port", type=int, required=True, help="server's port")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
def CreateSettings() -> Settings:
|
||||
arguments = ParseArguments()
|
||||
|
||||
settings = Settings()
|
||||
settings.host = arguments.host
|
||||
settings.port = arguments.port
|
||||
|
||||
return settings
|
||||
|
||||
def main():
|
||||
settings = CreateSettings()
|
||||
command_manager = CreateCommandManager(settings)
|
||||
|
||||
parameters = {}
|
||||
|
||||
PrintHelp(command_manager)
|
||||
|
||||
while True:
|
||||
command_name = str(input("> "))
|
||||
if command_name in ["q", "quit", "exit"]:
|
||||
break
|
||||
|
||||
if command_name in ["h", "help"]:
|
||||
PrintHelp(command_manager)
|
||||
continue
|
||||
|
||||
if not command_manager.hasCommand(command_name):
|
||||
print("there is no command with such name")
|
||||
continue
|
||||
|
||||
command = command_manager.getCommand(command_name)
|
||||
|
||||
try:
|
||||
command.do(parameters)
|
||||
except Exception as e:
|
||||
print(f"error: {str(e)}")
|
||||
|
||||
print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,19 +0,0 @@
|
||||
from modules.pipelines.pipe_manager import PipeManager
|
||||
from modules.pipelines.pipeline_manager import PipelineManager
|
||||
from modules.handlers.handler_manager import HandlerManager
|
||||
from modules.storages.storage import Storage
|
||||
|
||||
class Context:
|
||||
"""
|
||||
Контекст управляющего сервиса
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
self.pipe_manager = None
|
||||
self.pipeline_manager = None
|
||||
self.handler_manager = None
|
||||
self.storage = None
|
||||
@@ -1,130 +0,0 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from modules.logging.logger import *
|
||||
from modules.storages.storage import Storage
|
||||
from modules.transports.server import Server
|
||||
from services.control.context import *
|
||||
from services.control.create_context import *
|
||||
from services.control.response import *
|
||||
|
||||
def Handle(context, request):
|
||||
"""
|
||||
Обработка запроса
|
||||
:param context: Контекст сервиса
|
||||
:param request: Запрос
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
# Парсим запрос
|
||||
try:
|
||||
request = json.loads(request)
|
||||
except Exception as e:
|
||||
return ErrorResponse("invalid request", "failed to parse request")
|
||||
|
||||
# Проверяем наличие нужных ключей
|
||||
for key in ["handler", "parameters"]:
|
||||
if key not in request:
|
||||
return ErrorResponse("invalid request", f"key '{key}' is required")
|
||||
|
||||
# Проверяем наличие обработчика с указанным именем
|
||||
handler_name = request["handler"]
|
||||
if not context.handler_manager.contains(handler_name):
|
||||
return ErrorResponse("not found", f"there is no handler with name {handler_name}")
|
||||
|
||||
# Достаём нужного обработчика
|
||||
handler = context.handler_manager.get(handler_name)
|
||||
|
||||
# Запускаем обработку
|
||||
try:
|
||||
result = handler.do(context, request["parameters"])
|
||||
return SuccessResponse(result)
|
||||
except Exception as e:
|
||||
return ErrorResponse("handling failed", str(e))
|
||||
|
||||
|
||||
def ParseArguments():
|
||||
parser = argparse.ArgumentParser(description="Console client for Log Analysis")
|
||||
|
||||
# Данные сервера
|
||||
parser.add_argument("--host", type=str, required=True, help="server's host")
|
||||
parser.add_argument("--port", type=int, required=True, help="server's port")
|
||||
|
||||
# Данные для подключения к базе данных
|
||||
parser.add_argument("--psql-host", type=str, required=True, help="postgresql's host")
|
||||
parser.add_argument("--psql-port", type=str, required=True, help="postgresql's port")
|
||||
parser.add_argument("--psql-database", type=str, required=True, help="postgresql's database")
|
||||
parser.add_argument("--psql-user", type=str, required=True, help="postgresql's user")
|
||||
parser.add_argument("--psql-password", type=str, required=True, help="postgresql's password")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
class PostgreSqlSettings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.database = None
|
||||
self.user = None
|
||||
self.password = None
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.postgresql_settings = None
|
||||
|
||||
def CreateSettings():
|
||||
arguments = ParseArguments()
|
||||
|
||||
postgresql_settings = PostgreSqlSettings()
|
||||
postgresql_settings.host = arguments.psql_host
|
||||
postgresql_settings.port = arguments.psql_port
|
||||
postgresql_settings.database = arguments.psql_database
|
||||
postgresql_settings.user = arguments.psql_user
|
||||
postgresql_settings.password = arguments.psql_password
|
||||
|
||||
settings = Settings()
|
||||
settings.host = arguments.host
|
||||
settings.port = arguments.port
|
||||
settings.postgresql_settings = postgresql_settings
|
||||
|
||||
return settings
|
||||
|
||||
def LogCallback(level, message):
|
||||
print(f"level: {level}, message: {message}")
|
||||
|
||||
def main():
|
||||
settings = CreateSettings()
|
||||
|
||||
server = Server(settings.host, settings.port, Handle)
|
||||
server.setLogger(LogLevel.Warning, LogCallback)
|
||||
|
||||
# Создаём контекст
|
||||
context = CreateContext()
|
||||
|
||||
# Подключаемся к хранилищу
|
||||
context.storage = Storage(
|
||||
settings.postgresql_settings.host,
|
||||
settings.postgresql_settings.port,
|
||||
settings.postgresql_settings.database,
|
||||
settings.postgresql_settings.user,
|
||||
settings.postgresql_settings.password
|
||||
)
|
||||
|
||||
# Заполняем пайплайны
|
||||
query = "select name, pipes from pipelines;"
|
||||
pipelines = context.storage.do(query, should_return=True)
|
||||
|
||||
for name, pipes in pipelines:
|
||||
context.pipeline_manager.create(name, json.loads(pipes))
|
||||
|
||||
# Указываем контекст в сервере
|
||||
server.context = context
|
||||
|
||||
# Запускаем сервер
|
||||
asyncio.run(server.run())
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,62 +0,0 @@
|
||||
from modules.pipelines.pipe_manager import PipeManager
|
||||
from modules.pipelines.pipeline_manager import PipelineManager
|
||||
from modules.pipelines.pipes.preprocessing import *
|
||||
|
||||
from modules.handlers.handler_manager import HandlerManager
|
||||
from modules.handlers.pipelines.create_pipeline import CreatePipeline
|
||||
from modules.handlers.pipelines.get_pipelines import GetPipelines
|
||||
from modules.handlers.pipelines.get_pipes import GetPipes
|
||||
from modules.handlers.pipelines.run_pipeline import RunPipeline
|
||||
|
||||
from services.control.context import Context
|
||||
|
||||
def CreatePipeManager() -> PipeManager:
|
||||
"""
|
||||
Создание заполненного менеджера пайп
|
||||
:return: Менеджер пайп
|
||||
"""
|
||||
|
||||
manager = PipeManager()
|
||||
manager.add(RemovePunctuationMarks())
|
||||
manager.add(RemoveStopWords())
|
||||
manager.add(Stem())
|
||||
manager.add(ToLowerCase())
|
||||
manager.add(SplitBySpace())
|
||||
|
||||
return manager
|
||||
|
||||
def CreatePipelineManager(pipe_manager) -> PipelineManager:
|
||||
"""
|
||||
Создание заполненного менеджера пайплайнов
|
||||
:param pipe_manager: Менеджер пайп
|
||||
:return: Менеджер пайплайнов
|
||||
"""
|
||||
|
||||
manager = PipelineManager(pipe_manager)
|
||||
return manager
|
||||
|
||||
def CreateHandlerManager() -> HandlerManager:
|
||||
"""
|
||||
Создание менеджера хэндлеров
|
||||
:return: Менеджер хэндлеров
|
||||
"""
|
||||
manager = HandlerManager()
|
||||
manager.add(CreatePipeline())
|
||||
manager.add(GetPipelines())
|
||||
manager.add(GetPipes())
|
||||
manager.add(RunPipeline())
|
||||
|
||||
return manager
|
||||
|
||||
def CreateContext() -> Context:
|
||||
"""
|
||||
Создание заполненного контекста
|
||||
:return: Контекст
|
||||
"""
|
||||
|
||||
context = Context()
|
||||
context.pipe_manager = CreatePipeManager()
|
||||
context.pipeline_manager = CreatePipelineManager(context.pipe_manager)
|
||||
context.handler_manager = CreateHandlerManager()
|
||||
|
||||
return context
|
||||
@@ -1,26 +0,0 @@
|
||||
import json
|
||||
|
||||
def SuccessResponse(result: str) -> str:
|
||||
"""
|
||||
Создание успешного ответа
|
||||
:param result: Тело ответа
|
||||
:return: Ответ
|
||||
"""
|
||||
|
||||
return json.dumps({
|
||||
"status": "ok",
|
||||
"result": result
|
||||
})
|
||||
|
||||
def ErrorResponse(status: str, message: str) -> str:
|
||||
"""
|
||||
Создание ответа с ошибкой
|
||||
:param status: Код ошибки
|
||||
:param message: Сообщение об ошибке
|
||||
:return:
|
||||
"""
|
||||
|
||||
return json.dumps({
|
||||
"status": status,
|
||||
"message": message
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import re
|
||||
|
||||
from modules.logging.logger import LogLevel
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.dispatchers.dispatcher_base import DispatcherBase
|
||||
|
||||
def ConsumeCallback(context, message):
|
||||
match = re.search(str(context.pattern), message)
|
||||
if match is None:
|
||||
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
|
||||
return
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
|
||||
context.producer.send(topic, message)
|
||||
|
||||
class AllDispatcher(DispatcherBase):
|
||||
def __init__(self):
|
||||
# Паттерн: <number>month number hh:mm:ss machine-name process-name: message
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
super().__init__(topic="all", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for all logs")
|
||||
|
||||
def main():
|
||||
dispatcher = AllDispatcher()
|
||||
dispatcher.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
import re
|
||||
|
||||
from modules.logging.logger import LogLevel
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.dispatchers.dispatcher_base import DispatcherBase
|
||||
|
||||
def ConsumeCallback(context, message):
|
||||
match = re.search(str(context.pattern), message)
|
||||
if match is None:
|
||||
context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'")
|
||||
return
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[3]
|
||||
message = groups[4]
|
||||
|
||||
context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'")
|
||||
context.producer.send(topic, message)
|
||||
|
||||
class CephOsdDispatcher(DispatcherBase):
|
||||
def __init__(self):
|
||||
# Паттерн: yyyy-mm-ddThh:mm:ss+timezone pid number process-name: message
|
||||
datetime = Constants.number + "-" + Constants.number + "-" + Constants.number + "T" + Constants.time + "\." + Constants.number + "\+" + Constants.number
|
||||
datetime = datetime.replace("(", "")
|
||||
datetime = datetime.replace(")", "")
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
("(" + datetime + ")", "Date and time with timezone"), # 0
|
||||
(Constants.word, "pid"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.word + ":", "process name"), # 3
|
||||
(Constants.anything, "message") # 4
|
||||
]
|
||||
|
||||
super().__init__(topic="ceph-osd", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for ceph-osd logs")
|
||||
|
||||
def main():
|
||||
dispatcher = CephOsdDispatcher()
|
||||
dispatcher.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
from modules.logging.logger import LogLevel, Logger
|
||||
from modules.message_broker.consumer import Config as ConsumerConfig, Consumer
|
||||
from modules.message_broker.producer import Config as ProducerConfig, Producer
|
||||
from modules.utils.pattern import Pattern
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.logger = None
|
||||
self.pattern = None
|
||||
self.producer = None
|
||||
|
||||
def LogCallback(level, message):
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f"[{str(level)}] {now}: {message}")
|
||||
|
||||
def ParseArguments(description):
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument("--server", type=str, required=True, help="Kafka server")
|
||||
parser.add_argument("--pool-size", type=int, required=True, help="Pool size")
|
||||
parser.add_argument("--pool-cache-limit", type=int, required=True, help="Pool cache limit")
|
||||
parser.add_argument("--max-log-level", type=int, required=False, help="Max log level. Error -- 0 (default), Warning -- 1, Debug -- 2")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
return arguments
|
||||
|
||||
class DispatcherBase:
|
||||
def __init__(self, topic: str, pattern: Pattern, consume_callback, description: str):
|
||||
# Парсим аргументы
|
||||
arguments = ParseArguments(description)
|
||||
|
||||
# Создаём контекст
|
||||
self.context = Context()
|
||||
self.context.pattern = pattern
|
||||
|
||||
# Настраиваем логирование
|
||||
max_log_level = LogLevel.Debug # todo log level
|
||||
self.context.logger = Logger(max_log_level=max_log_level, callback=LogCallback)
|
||||
|
||||
# Создаём продюсера
|
||||
producer_config = ProducerConfig(arguments.server)
|
||||
self.context.producer = Producer(producer_config)
|
||||
|
||||
# Создаём консюмера
|
||||
consumer_config = ConsumerConfig(topic, arguments.server, arguments.pool_size, arguments.pool_cache_limit)
|
||||
self.consumer = Consumer(consumer_config, consume_callback)
|
||||
|
||||
def run(self):
|
||||
self.consumer.run(self.context)
|
||||
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
@@ -0,0 +1,138 @@
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
|
||||
from modules.message_broker.consumer import Config, Consumer
|
||||
from modules.storages.storage import ConnectionData, Storage
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.log_analyzer.model import LogAnalyzer
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.kafka_server = None
|
||||
self.topic = None
|
||||
self.model = None
|
||||
self.storage = None
|
||||
self.message_pattern = None
|
||||
self.enable_logging = False
|
||||
|
||||
def CheckRequiredKeys(config):
|
||||
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
|
||||
|
||||
is_ok = True
|
||||
for key in keys:
|
||||
if key not in config:
|
||||
print(f"config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
storage_config = config["storage"]
|
||||
storage_keys = ["host", "port", "database", "user", "password"]
|
||||
for key in storage_keys:
|
||||
if key not in storage_config:
|
||||
print(f"storage config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
def ParseConfig() -> Context:
|
||||
parser = argparse.ArgumentParser("Log analyzer")
|
||||
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
|
||||
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
with open(arguments.config, "r") as file:
|
||||
config_data = file.read()
|
||||
|
||||
config = json.loads(config_data)
|
||||
CheckRequiredKeys(config)
|
||||
|
||||
context = Context()
|
||||
context.kafka_server = config["kafka_server"]
|
||||
context.topic = config["topic"]
|
||||
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
|
||||
context.model.fit(config["model"])
|
||||
|
||||
storage_config = config["storage"]
|
||||
connection_data = ConnectionData()
|
||||
connection_data.host = storage_config["host"]
|
||||
connection_data.port = storage_config["port"]
|
||||
connection_data.database = storage_config["database"]
|
||||
connection_data.user = storage_config["user"]
|
||||
connection_data.password = storage_config["password"]
|
||||
|
||||
context.storage = Storage(connection_data)
|
||||
|
||||
context.enable_logging = arguments.enable_logging
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
context.message_pattern = re.compile(str(pattern))
|
||||
|
||||
return context
|
||||
|
||||
def ClearMessage(context: Context, message: str) -> tuple:
|
||||
match = re.search(context.message_pattern, message)
|
||||
if match is None:
|
||||
if context.enable_logging:
|
||||
print(f"Nothing is matched for log '{message}'")
|
||||
|
||||
return "", message
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
return topic, message
|
||||
|
||||
def UpdateStatisticsQuery(name: str):
|
||||
return f"update statistics set value = value + 1 where name = '{name}';"
|
||||
|
||||
def SaveAnomalyLogQuery(log: str):
|
||||
return f"insert into anomaly_logs (data) values('{log}')"
|
||||
|
||||
def Callback(context: Context, message: str):
|
||||
if context.enable_logging:
|
||||
print(f"Processing message '{message}'", message)
|
||||
|
||||
topic, message = ClearMessage(context, message)
|
||||
|
||||
result = context.model.predict(program=topic, message=message)[0]
|
||||
result_as_str = "anomaly" if result == -1 else "not anomaly"
|
||||
|
||||
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
|
||||
context.storage.do(query)
|
||||
|
||||
if result == -1:
|
||||
query = SaveAnomalyLogQuery(message)
|
||||
context.storage.do(query)
|
||||
|
||||
if context.enable_logging:
|
||||
print("log is " + result_as_str)
|
||||
|
||||
def main():
|
||||
# Создаём контекст
|
||||
context = ParseConfig()
|
||||
|
||||
# Создаём потребителя
|
||||
config = Config(context.topic, context.kafka_server, 16, 16)
|
||||
consumer = Consumer(config, Callback)
|
||||
|
||||
# Запускаем обработку
|
||||
consumer.run(context)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
from transformers import AutoTokenizer, AutoModel, AutoConfig
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import pickle
|
||||
import numpy
|
||||
import pandas as pd
|
||||
|
||||
class LogAnalyzer:
|
||||
'''Сlass of an expert system module that analyzes diagnostic information
|
||||
about the operation of the input-output subsystem.
|
||||
'''
|
||||
|
||||
def __init__(self, path_model_embedding: str, path_tokenizer: str):
|
||||
'''Constructor of model for analyse.
|
||||
:param path_model_embedding: String of path to directory with model
|
||||
for build embeddings of message
|
||||
:param path_tokenizer: String of path to directory with tokenizer
|
||||
for tokenize message
|
||||
'''
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
|
||||
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
|
||||
self.model = None
|
||||
|
||||
def predict(self, program: str, message: str) -> numpy.ndarray:
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param message: string of message
|
||||
'''
|
||||
|
||||
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
|
||||
'''Function applies mean pooling on the embedding model to get
|
||||
a single vector representation.
|
||||
:param model_output: Tuple containing different model outputs.
|
||||
The first element contains the token embeddings.
|
||||
:param attention_mask: Tensor that identifies the tokens
|
||||
to which attention should be paid (value of 1)
|
||||
versus those which should not (value of 0).
|
||||
:return: Tensor representing the pooled output of the input tokens.
|
||||
'''
|
||||
token_embeddings = model_output[0]
|
||||
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
|
||||
token_embeddings.size()).float()
|
||||
return torch.sum(
|
||||
token_embeddings * input_mask_expanded, 1) / torch.clamp(
|
||||
input_mask_expanded.sum(1), min=1e-9)
|
||||
|
||||
def embed(message: str, tokenizer, model) -> torch.Tensor:
|
||||
'''Function generates embeddings for a given message.
|
||||
:param message: String of message for which embeddings
|
||||
are to be generated.
|
||||
:param tokenizer: The tokenizer to be used for tokenizing
|
||||
the message.
|
||||
:param model: The model to be used for generating embeddings.
|
||||
:return: Tensor of normalized embeddings of the input message.
|
||||
'''
|
||||
encoded_input = tokenizer(
|
||||
message, padding=True, truncation=True, return_tensors='pt')
|
||||
|
||||
with torch.no_grad():
|
||||
model_output = model(**encoded_input)
|
||||
|
||||
message_embeddings_mean = mean_pooling(
|
||||
model_output, encoded_input['attention_mask'])
|
||||
message_embeddings = F.normalize(
|
||||
message_embeddings_mean, p=2, dim=1)
|
||||
return message_embeddings
|
||||
|
||||
if self.model is None:
|
||||
raise ValueError("Model is not fitted. Make '.fit' method first")
|
||||
return None
|
||||
|
||||
message_embeddings = embed(
|
||||
message, self.tokenizer, self.model_embedding)
|
||||
|
||||
try:
|
||||
prediction = self.model[program].predict(
|
||||
message_embeddings.numpy())
|
||||
except KeyError:
|
||||
prediction = self.model['unknown'].predict(
|
||||
message_embeddings.numpy())
|
||||
return prediction
|
||||
|
||||
def fit(self, path_model):
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param path_model: String of path to saved model
|
||||
'''
|
||||
self.model = pickle.load(open(path_model, 'rb'))
|
||||
@@ -0,0 +1,33 @@
|
||||
import re
|
||||
|
||||
from modules.pipelines.pipes.preprocessing import DataType, PipeBase
|
||||
|
||||
class RemoveByRegex(PipeBase):
|
||||
def __init__(self, name, pattern):
|
||||
super().__init__(name, DataType.String, DataType.String)
|
||||
|
||||
self.pattern = re.compile(pattern)
|
||||
|
||||
def do(self, data):
|
||||
return self.pattern.sub("", data)
|
||||
|
||||
class RemoveOriginalLogTime(RemoveByRegex):
|
||||
def __init__(self):
|
||||
super().__init__("remove_original_log_time", "\(Original Log Time .*\)")
|
||||
|
||||
class RemoveSourceCode(RemoveByRegex):
|
||||
def __init__(self):
|
||||
super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]")
|
||||
|
||||
class StopRunningIfContains(PipeBase):
|
||||
def __init__(self, items: list[str]):
|
||||
super().__init__("stop_running_if_contains", DataType.String, DataType.String)
|
||||
|
||||
self.items = items
|
||||
|
||||
def do(self, data: str):
|
||||
for item in self.items:
|
||||
if data.find(item) != -1:
|
||||
return None
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,22 @@
|
||||
from modules.pipelines.pipeline import Pipeline
|
||||
from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace
|
||||
from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains
|
||||
|
||||
def main():
|
||||
pipeline = Pipeline("prepare")
|
||||
pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"]))
|
||||
pipeline.add(RemoveOriginalLogTime())
|
||||
pipeline.add(RemoveSourceCode())
|
||||
pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]"))
|
||||
pipeline.add(Strip())
|
||||
|
||||
with open("rocksdb_logs.txt", "r") as input:
|
||||
with open("filtered_rocksdb_logs.txt", "w") as output:
|
||||
for line in input.readlines():
|
||||
result = pipeline.run(line)
|
||||
|
||||
if result is not None:
|
||||
output.write(result + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import argparse
|
||||
from kafka import KafkaConsumer
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--topic", type=str, required=True)
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
consumer = KafkaConsumer(bootstrap_servers=["localhost:29092"])
|
||||
consumer.subscribe(arguments.topic)
|
||||
|
||||
for message in consumer:
|
||||
print(message.value)
|
||||
@@ -0,0 +1,12 @@
|
||||
import argparse
|
||||
from kafka import KafkaProducer
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--topic", type=str, required=True)
|
||||
parser.add_argument("--message", type=str, required=True)
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
producer = KafkaProducer(bootstrap_servers=["localhost:29092"])
|
||||
future = producer.send(arguments.topic, arguments.message.encode())
|
||||
future.get(timeout=3)
|
||||
@@ -1,7 +0,0 @@
|
||||
create database log_analysis;
|
||||
|
||||
create table pipelines(
|
||||
id serial,
|
||||
name text,
|
||||
pipes text
|
||||
);
|
||||
@@ -0,0 +1,15 @@
|
||||
create database log_analysis;
|
||||
|
||||
create table statistics(
|
||||
name varchar(64),
|
||||
value int,
|
||||
primary key (name)
|
||||
);
|
||||
|
||||
insert into statistics (name, value) values ('total', 0), ('anomaly', 0), ('not anomaly', 0), ('undefined', 0);
|
||||
|
||||
create table anomaly_logs(
|
||||
id serial,
|
||||
data text,
|
||||
primary key (id)
|
||||
);
|
||||
Reference in New Issue
Block a user