Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+4
View File
@@ -0,0 +1,4 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+138
View File
@@ -0,0 +1,138 @@
import argparse
import json
import re
from modules.message_broker.consumer import Config, Consumer
from modules.storages.storage import ConnectionData, Storage
from modules.utils.pattern import Constants, Pattern
from services.log_analyzer.model import LogAnalyzer
class Context:
def __init__(self):
self.kafka_server = None
self.topic = None
self.model = None
self.storage = None
self.message_pattern = None
self.enable_logging = False
def CheckRequiredKeys(config):
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
is_ok = True
for key in keys:
if key not in config:
print(f"config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
storage_config = config["storage"]
storage_keys = ["host", "port", "database", "user", "password"]
for key in storage_keys:
if key not in storage_config:
print(f"storage config doesn't contain key '{key}'")
is_ok = False
if not is_ok:
raise RuntimeError("Invalid configuration file")
def ParseConfig() -> Context:
parser = argparse.ArgumentParser("Log analyzer")
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
arguments = parser.parse_args()
with open(arguments.config, "r") as file:
config_data = file.read()
config = json.loads(config_data)
CheckRequiredKeys(config)
context = Context()
context.kafka_server = config["kafka_server"]
context.topic = config["topic"]
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
context.model.fit(config["model"])
storage_config = config["storage"]
connection_data = ConnectionData()
connection_data.host = storage_config["host"]
connection_data.port = storage_config["port"]
connection_data.database = storage_config["database"]
connection_data.user = storage_config["user"]
connection_data.password = storage_config["password"]
context.storage = Storage(connection_data)
context.enable_logging = arguments.enable_logging
pattern = Pattern()
pattern.parts = [
(Constants.unknown_begin_part, "unknown begin part"), # 0
(Constants.month, "month"), # 1
(Constants.number, "unknown number"), # 2
(Constants.time, "time"), # 3
(Constants.word, "machine name"), # 4
(Constants.word + ":", "process name"), # 5
(Constants.anything, "message") # 6
]
context.message_pattern = re.compile(str(pattern))
return context
def ClearMessage(context: Context, message: str) -> tuple:
match = re.search(context.message_pattern, message)
if match is None:
if context.enable_logging:
print(f"Nothing is matched for log '{message}'")
return "", message
groups = match.groups()
topic = groups[5]
message = groups[6]
return topic, message
def UpdateStatisticsQuery(name: str):
return f"update statistics set value = value + 1 where name = '{name}';"
def SaveAnomalyLogQuery(log: str):
return f"insert into anomaly_logs (data) values('{log}')"
def Callback(context: Context, message: str):
if context.enable_logging:
print(f"Processing message '{message}'", message)
topic, message = ClearMessage(context, message)
result = context.model.predict(program=topic, message=message)[0]
result_as_str = "anomaly" if result == -1 else "not anomaly"
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
context.storage.do(query)
if result == -1:
query = SaveAnomalyLogQuery(message)
context.storage.do(query)
if context.enable_logging:
print("log is " + result_as_str)
def main():
# Создаём контекст
context = ParseConfig()
# Создаём потребителя
config = Config(context.topic, context.kafka_server, 16, 16)
consumer = Consumer(config, Callback)
# Запускаем обработку
consumer.run(context)
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
from transformers import AutoTokenizer, AutoModel, AutoConfig
import torch
import torch.nn.functional as F
import pickle
import numpy
import pandas as pd
class LogAnalyzer:
'''Сlass of an expert system module that analyzes diagnostic information
about the operation of the input-output subsystem.
'''
def __init__(self, path_model_embedding: str, path_tokenizer: str):
'''Constructor of model for analyse.
:param path_model_embedding: String of path to directory with model
for build embeddings of message
:param path_tokenizer: String of path to directory with tokenizer
for tokenize message
'''
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
self.model = None
def predict(self, program: str, message: str) -> numpy.ndarray:
'''Function predict whether an log is anomaly.
:param message: string of message
'''
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
'''Function applies mean pooling on the embedding model to get
a single vector representation.
:param model_output: Tuple containing different model outputs.
The first element contains the token embeddings.
:param attention_mask: Tensor that identifies the tokens
to which attention should be paid (value of 1)
versus those which should not (value of 0).
:return: Tensor representing the pooled output of the input tokens.
'''
token_embeddings = model_output[0]
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
token_embeddings.size()).float()
return torch.sum(
token_embeddings * input_mask_expanded, 1) / torch.clamp(
input_mask_expanded.sum(1), min=1e-9)
def embed(message: str, tokenizer, model) -> torch.Tensor:
'''Function generates embeddings for a given message.
:param message: String of message for which embeddings
are to be generated.
:param tokenizer: The tokenizer to be used for tokenizing
the message.
:param model: The model to be used for generating embeddings.
:return: Tensor of normalized embeddings of the input message.
'''
encoded_input = tokenizer(
message, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
model_output = model(**encoded_input)
message_embeddings_mean = mean_pooling(
model_output, encoded_input['attention_mask'])
message_embeddings = F.normalize(
message_embeddings_mean, p=2, dim=1)
return message_embeddings
if self.model is None:
raise ValueError("Model is not fitted. Make '.fit' method first")
return None
message_embeddings = embed(
message, self.tokenizer, self.model_embedding)
try:
prediction = self.model[program].predict(
message_embeddings.numpy())
except KeyError:
prediction = self.model['unknown'].predict(
message_embeddings.numpy())
return prediction
def fit(self, path_model):
'''Function predict whether an log is anomaly.
:param path_model: String of path to saved model
'''
self.model = pickle.load(open(path_model, 'rb'))