Большое обновление всего проекта
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
@@ -0,0 +1,138 @@
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
|
||||
from modules.message_broker.consumer import Config, Consumer
|
||||
from modules.storages.storage import ConnectionData, Storage
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.log_analyzer.model import LogAnalyzer
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.kafka_server = None
|
||||
self.topic = None
|
||||
self.model = None
|
||||
self.storage = None
|
||||
self.message_pattern = None
|
||||
self.enable_logging = False
|
||||
|
||||
def CheckRequiredKeys(config):
|
||||
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
|
||||
|
||||
is_ok = True
|
||||
for key in keys:
|
||||
if key not in config:
|
||||
print(f"config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
storage_config = config["storage"]
|
||||
storage_keys = ["host", "port", "database", "user", "password"]
|
||||
for key in storage_keys:
|
||||
if key not in storage_config:
|
||||
print(f"storage config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
def ParseConfig() -> Context:
|
||||
parser = argparse.ArgumentParser("Log analyzer")
|
||||
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
|
||||
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
with open(arguments.config, "r") as file:
|
||||
config_data = file.read()
|
||||
|
||||
config = json.loads(config_data)
|
||||
CheckRequiredKeys(config)
|
||||
|
||||
context = Context()
|
||||
context.kafka_server = config["kafka_server"]
|
||||
context.topic = config["topic"]
|
||||
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
|
||||
context.model.fit(config["model"])
|
||||
|
||||
storage_config = config["storage"]
|
||||
connection_data = ConnectionData()
|
||||
connection_data.host = storage_config["host"]
|
||||
connection_data.port = storage_config["port"]
|
||||
connection_data.database = storage_config["database"]
|
||||
connection_data.user = storage_config["user"]
|
||||
connection_data.password = storage_config["password"]
|
||||
|
||||
context.storage = Storage(connection_data)
|
||||
|
||||
context.enable_logging = arguments.enable_logging
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
context.message_pattern = re.compile(str(pattern))
|
||||
|
||||
return context
|
||||
|
||||
def ClearMessage(context: Context, message: str) -> tuple:
|
||||
match = re.search(context.message_pattern, message)
|
||||
if match is None:
|
||||
if context.enable_logging:
|
||||
print(f"Nothing is matched for log '{message}'")
|
||||
|
||||
return "", message
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
return topic, message
|
||||
|
||||
def UpdateStatisticsQuery(name: str):
|
||||
return f"update statistics set value = value + 1 where name = '{name}';"
|
||||
|
||||
def SaveAnomalyLogQuery(log: str):
|
||||
return f"insert into anomaly_logs (data) values('{log}')"
|
||||
|
||||
def Callback(context: Context, message: str):
|
||||
if context.enable_logging:
|
||||
print(f"Processing message '{message}'", message)
|
||||
|
||||
topic, message = ClearMessage(context, message)
|
||||
|
||||
result = context.model.predict(program=topic, message=message)[0]
|
||||
result_as_str = "anomaly" if result == -1 else "not anomaly"
|
||||
|
||||
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
|
||||
context.storage.do(query)
|
||||
|
||||
if result == -1:
|
||||
query = SaveAnomalyLogQuery(message)
|
||||
context.storage.do(query)
|
||||
|
||||
if context.enable_logging:
|
||||
print("log is " + result_as_str)
|
||||
|
||||
def main():
|
||||
# Создаём контекст
|
||||
context = ParseConfig()
|
||||
|
||||
# Создаём потребителя
|
||||
config = Config(context.topic, context.kafka_server, 16, 16)
|
||||
consumer = Consumer(config, Callback)
|
||||
|
||||
# Запускаем обработку
|
||||
consumer.run(context)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
from transformers import AutoTokenizer, AutoModel, AutoConfig
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import pickle
|
||||
import numpy
|
||||
import pandas as pd
|
||||
|
||||
class LogAnalyzer:
|
||||
'''Сlass of an expert system module that analyzes diagnostic information
|
||||
about the operation of the input-output subsystem.
|
||||
'''
|
||||
|
||||
def __init__(self, path_model_embedding: str, path_tokenizer: str):
|
||||
'''Constructor of model for analyse.
|
||||
:param path_model_embedding: String of path to directory with model
|
||||
for build embeddings of message
|
||||
:param path_tokenizer: String of path to directory with tokenizer
|
||||
for tokenize message
|
||||
'''
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
|
||||
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
|
||||
self.model = None
|
||||
|
||||
def predict(self, program: str, message: str) -> numpy.ndarray:
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param message: string of message
|
||||
'''
|
||||
|
||||
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
|
||||
'''Function applies mean pooling on the embedding model to get
|
||||
a single vector representation.
|
||||
:param model_output: Tuple containing different model outputs.
|
||||
The first element contains the token embeddings.
|
||||
:param attention_mask: Tensor that identifies the tokens
|
||||
to which attention should be paid (value of 1)
|
||||
versus those which should not (value of 0).
|
||||
:return: Tensor representing the pooled output of the input tokens.
|
||||
'''
|
||||
token_embeddings = model_output[0]
|
||||
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
|
||||
token_embeddings.size()).float()
|
||||
return torch.sum(
|
||||
token_embeddings * input_mask_expanded, 1) / torch.clamp(
|
||||
input_mask_expanded.sum(1), min=1e-9)
|
||||
|
||||
def embed(message: str, tokenizer, model) -> torch.Tensor:
|
||||
'''Function generates embeddings for a given message.
|
||||
:param message: String of message for which embeddings
|
||||
are to be generated.
|
||||
:param tokenizer: The tokenizer to be used for tokenizing
|
||||
the message.
|
||||
:param model: The model to be used for generating embeddings.
|
||||
:return: Tensor of normalized embeddings of the input message.
|
||||
'''
|
||||
encoded_input = tokenizer(
|
||||
message, padding=True, truncation=True, return_tensors='pt')
|
||||
|
||||
with torch.no_grad():
|
||||
model_output = model(**encoded_input)
|
||||
|
||||
message_embeddings_mean = mean_pooling(
|
||||
model_output, encoded_input['attention_mask'])
|
||||
message_embeddings = F.normalize(
|
||||
message_embeddings_mean, p=2, dim=1)
|
||||
return message_embeddings
|
||||
|
||||
if self.model is None:
|
||||
raise ValueError("Model is not fitted. Make '.fit' method first")
|
||||
return None
|
||||
|
||||
message_embeddings = embed(
|
||||
message, self.tokenizer, self.model_embedding)
|
||||
|
||||
try:
|
||||
prediction = self.model[program].predict(
|
||||
message_embeddings.numpy())
|
||||
except KeyError:
|
||||
prediction = self.model['unknown'].predict(
|
||||
message_embeddings.numpy())
|
||||
return prediction
|
||||
|
||||
def fit(self, path_model):
|
||||
'''Function predict whether an log is anomaly.
|
||||
:param path_model: String of path to saved model
|
||||
'''
|
||||
self.model = pickle.load(open(path_model, 'rb'))
|
||||
Reference in New Issue
Block a user