Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+86
View File
@@ -0,0 +1,86 @@
from transformers import AutoTokenizer, AutoModel, AutoConfig
import torch
import torch.nn.functional as F
import pickle
import numpy
import pandas as pd
class LogAnalyzer:
'''Сlass of an expert system module that analyzes diagnostic information
about the operation of the input-output subsystem.
'''
def __init__(self, path_model_embedding: str, path_tokenizer: str):
'''Constructor of model for analyse.
:param path_model_embedding: String of path to directory with model
for build embeddings of message
:param path_tokenizer: String of path to directory with tokenizer
for tokenize message
'''
self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer)
self.model_embedding = AutoModel.from_pretrained(path_model_embedding)
self.model = None
def predict(self, program: str, message: str) -> numpy.ndarray:
'''Function predict whether an log is anomaly.
:param message: string of message
'''
def mean_pooling(model_output, attention_mask) -> torch.Tensor:
'''Function applies mean pooling on the embedding model to get
a single vector representation.
:param model_output: Tuple containing different model outputs.
The first element contains the token embeddings.
:param attention_mask: Tensor that identifies the tokens
to which attention should be paid (value of 1)
versus those which should not (value of 0).
:return: Tensor representing the pooled output of the input tokens.
'''
token_embeddings = model_output[0]
input_mask_expanded = attention_mask.unsqueeze(-1).expand(
token_embeddings.size()).float()
return torch.sum(
token_embeddings * input_mask_expanded, 1) / torch.clamp(
input_mask_expanded.sum(1), min=1e-9)
def embed(message: str, tokenizer, model) -> torch.Tensor:
'''Function generates embeddings for a given message.
:param message: String of message for which embeddings
are to be generated.
:param tokenizer: The tokenizer to be used for tokenizing
the message.
:param model: The model to be used for generating embeddings.
:return: Tensor of normalized embeddings of the input message.
'''
encoded_input = tokenizer(
message, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
model_output = model(**encoded_input)
message_embeddings_mean = mean_pooling(
model_output, encoded_input['attention_mask'])
message_embeddings = F.normalize(
message_embeddings_mean, p=2, dim=1)
return message_embeddings
if self.model is None:
raise ValueError("Model is not fitted. Make '.fit' method first")
return None
message_embeddings = embed(
message, self.tokenizer, self.model_embedding)
try:
prediction = self.model[program].predict(
message_embeddings.numpy())
except KeyError:
prediction = self.model['unknown'].predict(
message_embeddings.numpy())
return prediction
def fit(self, path_model):
'''Function predict whether an log is anomaly.
:param path_model: String of path to saved model
'''
self.model = pickle.load(open(path_model, 'rb'))