Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+18
View File
@@ -0,0 +1,18 @@
class Subscriber:
def __init__(self):
self.host = None
self.port = None
class Context:
"""
Контекст обработчиков
"""
def __init__(self):
"""
Конструктор
"""
self.pipe_manager = None
self.pipeline_manager = None
self.subscribers = {}
@@ -1,3 +1,5 @@
from modules.handlers.context import Context
def CheckParameters(parameters, required_keys):
for key in required_keys:
if key not in parameters:
@@ -7,5 +9,5 @@ class HandlerBase:
def __init__(self, name):
self.name = name
def do(self, context, parameters):
def do(self, context: Context, parameters: dict):
pass
+109
View File
@@ -0,0 +1,109 @@
import json
from modules.pipelines.pipes.pipe_base import ToString
from modules.handlers.handler_base import *
class CreatePipeline(HandlerBase):
"""
Обработчик создания пайплайна
"""
def __init__(self):
super().__init__("create_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "pipes"])
name = parameters["name"]
pipes = parameters["pipes"]
context.pipeline_manager.create(name, pipes)
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
context.storage.do(query)
class GetPipelines(HandlerBase):
"""
Обработчик получения списка пайплайнов
"""
def __init__(self):
super().__init__("get_pipelines")
def do(self, context: Context, parameters: dict):
result = []
for name, pipeline in context.pipeline_manager.pipelines.items():
item = {
"name": pipeline.name,
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
}
result.append(item)
return result
class GetPipes(HandlerBase):
"""
Обработчик получения списка пайп
"""
def __init__(self):
super().__init__("get_pipes")
def do(self, context: Context, parameters: dict):
result = []
for pipe_name, pipe in context.pipe_manager.pipes.items():
item = {
"name": pipe.name,
"description": pipe.description,
"input_data_type": ToString(pipe.input_data_type),
"output_data_type": ToString(pipe.output_data_type)
}
result.append(item)
return result
class RunPipeline(HandlerBase):
"""
Обработчик запуска пайплайна
"""
def __init__(self):
super().__init__("run_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "data"])
name = parameters["name"]
data = parameters["data"]
pipeline = context.pipeline_manager.get(name)
result = pipeline.run(data)
for subscriber in context.subscribers[name]:
try:
Send(subscriber.host, subscriber.port, result)
return result
class SubscribeOnPipeline(HandlerBase):
"""
Обработчик подписки на результаты работы пайплайна
"""
def __init__(self):
super().__init__("subscribe_on_pipeline")
def do(self, context: Context, parameters: dict):
CheckParameters(parameters, ["name", "host", "port"])
name = parameters["name"]
host = parameters["host"]
port = parameters["port"]
context.sub
return result
@@ -1,17 +0,0 @@
import json
from modules.handlers.base.handler_base import *
class CreatePipeline(HandlerBase):
def __init__(self):
super().__init__("create_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "pipes"])
name = parameters["name"]
pipes = parameters["pipes"]
context.pipeline_manager.create(name, pipes)
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
context.storage.do(query)
@@ -1,18 +0,0 @@
from modules.handlers.base.handler_base import HandlerBase
class GetPipelines(HandlerBase):
def __init__(self):
super().__init__("get_pipelines")
def do(self, context, parameters):
result = []
for name, pipeline in context.pipeline_manager.pipelines.items():
item = {
"name": pipeline.name,
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
}
result.append(item)
return result
-21
View File
@@ -1,21 +0,0 @@
from modules.pipelines.pipes.pipe_base import ToString
from modules.handlers.base.handler_base import HandlerBase
class GetPipes(HandlerBase):
def __init__(self):
super().__init__("get_pipes")
def do(self, context, parameters):
result = []
for pipe_name, pipe in context.pipe_manager.pipes.items():
item = {
"name": pipe.name,
"description": pipe.description,
"input_data_type": ToString(pipe.input_data_type),
"output_data_type": ToString(pipe.output_data_type)
}
result.append(item)
return result
@@ -1,16 +0,0 @@
from modules.handlers.base.handler_base import *
class RunPipeline(HandlerBase):
def __init__(self):
super().__init__("run_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "data"])
name = parameters["name"]
data = parameters["data"]
pipeline = context.pipeline_manager.get(name)
result = pipeline.run(data)
return result
+14 -3
View File
@@ -6,13 +6,24 @@ class LogLevel(Enum):
Debug = 2
def __int__(self):
return self.value
return self.value[0]
def __str__(self):
if self.value == LogLevel.Error:
return "Error"
elif self.value == LogLevel.Warning:
return "Warning"
elif self.value == LogLevel.Debug:
return "Debug"
else:
return "Unknown log level"
class Logger:
def __init__(self, max_log_level: LogLevel, callback):
self.max_log_level = max_log_level
self.callback = callback
def log(self, level, message):
if self.callback is not None and int(level) <= int(self.max_log_level):
def log(self, level: LogLevel, message):
# if self.callback is not None and int(level) <= int(self.max_log_level):
if self.callback:
self.callback(level, message)
+45
View File
@@ -0,0 +1,45 @@
import signal
import kafka
import kafka.errors
import multiprocessing.pool
class Config:
def __init__(self, topic, server, pool_size, pool_cache_limit):
self.topic = topic
self.server = server
self.pool_size = pool_size
self.pool_cache_limit = pool_cache_limit
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
def getPoolCacheSize(self):
return len(self._cache)
class Consumer:
def __init__(self, config: Config, callback):
self.callback = callback
self.consumer = kafka.KafkaConsumer(
config.topic,
bootstrap_servers=[config.server],
)
self.should_stop = False
self.pool_cache_limit = config.pool_cache_limit
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
signal.signal(signal.SIGTERM, self.stop)
def handlePoolCacheExcess(self):
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
pass
def run(self, context):
print("consumer is running")
while not self.should_stop:
for message in self.consumer:
if self.should_stop:
break
self.callback(context, message.value.decode())
def stop(self):
self.should_stop = True
+14
View File
@@ -0,0 +1,14 @@
import kafka
class Config:
def __init__(self, server):
self.server = server
class Producer:
def __init__(self, config):
self.producer = kafka.KafkaProducer(
bootstrap_servers=[config.server]
)
def send(self, topic: str, message: str):
self.producer.send(topic, message.encode())
+3
View File
@@ -20,4 +20,7 @@ class Pipeline:
for pipe in self.pipes:
data = pipe.do(data)
if data is None:
return None
return data
+16
View File
@@ -19,6 +19,22 @@ class ToLowerCase(PipeBase):
def do(self, data):
return data.lower()
class Strip(PipeBase):
"""
Удаляет пробелы в начале и конце строки
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("strip", DataType.String, DataType.String)
self.description = "removes any leading and trailing whitespaces"
def do(self, data: str):
return data.strip()
class Split(PipeBase):
"""
Разбивает строку на токены через разделитель
+10 -3
View File
@@ -1,8 +1,16 @@
import psycopg2 as pg
class ConnectionData:
def __init__(self):
self.host = None
self.port = None
self.database = None
self.user = None
self.password = None
class Storage:
def __init__(self, host, port, database, user, password):
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
def __init__(self, connection_data):
self.connection = pg.connect(host=connection_data.host, port=connection_data.port, dbname=connection_data.database, user=connection_data.user, password=connection_data.password)
def do(self, query, should_return=False):
transaction = self.connection.cursor()
@@ -12,4 +20,3 @@ class Storage:
if should_return:
return transaction.fetchall()
-6
View File
@@ -9,10 +9,6 @@ class Server:
self.callback = callback
self.context = None
self.server = None
self.logger = Logger(LogLevel.Error, None)
def setLogger(self, max_log_level, callback):
self.logger = Logger(max_log_level, callback)
async def run(self):
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
@@ -42,11 +38,9 @@ class Server:
request = request.decode()
# Вызываем коллбэк
response = self.callback(self.context, request)
response = struct.pack(">I", len(response)) + response.encode()
# Отправляем ответ
writer.write(response)
def stop(self):
@@ -0,0 +1,26 @@
class NgramInfo:
def __init__(self, id):
self.id = id
self.frequency = 1
class NgramsDictionary:
def __init__(self, ngram_size):
self.ngrams = {}
self.ngram_size = ngram_size
self.last_id = 0
def update(self, words):
for last_word_index in range(self.ngram_size, len(words) + 1):
ngram = ""
for current_word_index in range(0, self.ngram_size):
ngram += words[last_word_index - self.ngram_size + current_word_index] + " "
self.addNgram(ngram[:-1])
def addNgram(self, ngram):
if ngram in self.ngrams:
self.ngrams[ngram].frequency += 1
else:
word_info = NgramInfo(self.last_id)
self.last_id += 1
self.ngrams[ngram] = word_info
+20
View File
@@ -0,0 +1,20 @@
class Constants:
whitespace = r"\s*"
unknown_begin_part = r"<(\d{1,2})>"
month = r"([A-Za-z]{3})"
number = r"(\d+)"
time = r"(\d{2}:\d{2}:\d{2})"
word = r"(\S+)"
anything = r"(.*)"
class Pattern:
def __int__(self):
self.parts = []
def __str__(self):
result = ""
for part in self.parts:
result += part[0] + Constants.whitespace
return result