Большое обновление всего проекта
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
|
||||
from modules.message_broker.consumer import Config, Consumer
|
||||
from modules.storages.storage import ConnectionData, Storage
|
||||
from modules.utils.pattern import Constants, Pattern
|
||||
from services.log_analyzer.model import LogAnalyzer
|
||||
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.kafka_server = None
|
||||
self.topic = None
|
||||
self.model = None
|
||||
self.storage = None
|
||||
self.message_pattern = None
|
||||
self.enable_logging = False
|
||||
|
||||
def CheckRequiredKeys(config):
|
||||
keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"]
|
||||
|
||||
is_ok = True
|
||||
for key in keys:
|
||||
if key not in config:
|
||||
print(f"config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
storage_config = config["storage"]
|
||||
storage_keys = ["host", "port", "database", "user", "password"]
|
||||
for key in storage_keys:
|
||||
if key not in storage_config:
|
||||
print(f"storage config doesn't contain key '{key}'")
|
||||
is_ok = False
|
||||
|
||||
if not is_ok:
|
||||
raise RuntimeError("Invalid configuration file")
|
||||
|
||||
def ParseConfig() -> Context:
|
||||
parser = argparse.ArgumentParser("Log analyzer")
|
||||
parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file")
|
||||
parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
with open(arguments.config, "r") as file:
|
||||
config_data = file.read()
|
||||
|
||||
config = json.loads(config_data)
|
||||
CheckRequiredKeys(config)
|
||||
|
||||
context = Context()
|
||||
context.kafka_server = config["kafka_server"]
|
||||
context.topic = config["topic"]
|
||||
context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"])
|
||||
context.model.fit(config["model"])
|
||||
|
||||
storage_config = config["storage"]
|
||||
connection_data = ConnectionData()
|
||||
connection_data.host = storage_config["host"]
|
||||
connection_data.port = storage_config["port"]
|
||||
connection_data.database = storage_config["database"]
|
||||
connection_data.user = storage_config["user"]
|
||||
connection_data.password = storage_config["password"]
|
||||
|
||||
context.storage = Storage(connection_data)
|
||||
|
||||
context.enable_logging = arguments.enable_logging
|
||||
|
||||
pattern = Pattern()
|
||||
pattern.parts = [
|
||||
(Constants.unknown_begin_part, "unknown begin part"), # 0
|
||||
(Constants.month, "month"), # 1
|
||||
(Constants.number, "unknown number"), # 2
|
||||
(Constants.time, "time"), # 3
|
||||
(Constants.word, "machine name"), # 4
|
||||
(Constants.word + ":", "process name"), # 5
|
||||
(Constants.anything, "message") # 6
|
||||
]
|
||||
|
||||
context.message_pattern = re.compile(str(pattern))
|
||||
|
||||
return context
|
||||
|
||||
def ClearMessage(context: Context, message: str) -> tuple:
|
||||
match = re.search(context.message_pattern, message)
|
||||
if match is None:
|
||||
if context.enable_logging:
|
||||
print(f"Nothing is matched for log '{message}'")
|
||||
|
||||
return "", message
|
||||
|
||||
groups = match.groups()
|
||||
topic = groups[5]
|
||||
message = groups[6]
|
||||
|
||||
return topic, message
|
||||
|
||||
def UpdateStatisticsQuery(name: str):
|
||||
return f"update statistics set value = value + 1 where name = '{name}';"
|
||||
|
||||
def SaveAnomalyLogQuery(log: str):
|
||||
return f"insert into anomaly_logs (data) values('{log}')"
|
||||
|
||||
def Callback(context: Context, message: str):
|
||||
if context.enable_logging:
|
||||
print(f"Processing message '{message}'", message)
|
||||
|
||||
topic, message = ClearMessage(context, message)
|
||||
|
||||
result = context.model.predict(program=topic, message=message)[0]
|
||||
result_as_str = "anomaly" if result == -1 else "not anomaly"
|
||||
|
||||
query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str)
|
||||
context.storage.do(query)
|
||||
|
||||
if result == -1:
|
||||
query = SaveAnomalyLogQuery(message)
|
||||
context.storage.do(query)
|
||||
|
||||
if context.enable_logging:
|
||||
print("log is " + result_as_str)
|
||||
|
||||
def main():
|
||||
# Создаём контекст
|
||||
context = ParseConfig()
|
||||
|
||||
# Создаём потребителя
|
||||
config = Config(context.topic, context.kafka_server, 16, 16)
|
||||
consumer = Consumer(config, Callback)
|
||||
|
||||
# Запускаем обработку
|
||||
consumer.run(context)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user