Большое обновление всего проекта
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
class Subscriber:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
|
||||
class Context:
|
||||
"""
|
||||
Контекст обработчиков
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
self.pipe_manager = None
|
||||
self.pipeline_manager = None
|
||||
self.subscribers = {}
|
||||
@@ -1,3 +1,5 @@
|
||||
from modules.handlers.context import Context
|
||||
|
||||
def CheckParameters(parameters, required_keys):
|
||||
for key in required_keys:
|
||||
if key not in parameters:
|
||||
@@ -7,5 +9,5 @@ class HandlerBase:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def do(self, context, parameters):
|
||||
def do(self, context: Context, parameters: dict):
|
||||
pass
|
||||
@@ -0,0 +1,109 @@
|
||||
import json
|
||||
|
||||
from modules.pipelines.pipes.pipe_base import ToString
|
||||
from modules.handlers.handler_base import *
|
||||
|
||||
class CreatePipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик создания пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "pipes"])
|
||||
|
||||
name = parameters["name"]
|
||||
pipes = parameters["pipes"]
|
||||
|
||||
context.pipeline_manager.create(name, pipes)
|
||||
|
||||
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
|
||||
context.storage.do(query)
|
||||
|
||||
class GetPipelines(HandlerBase):
|
||||
"""
|
||||
Обработчик получения списка пайплайнов
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
result = []
|
||||
|
||||
for name, pipeline in context.pipeline_manager.pipelines.items():
|
||||
item = {
|
||||
"name": pipeline.name,
|
||||
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
|
||||
class GetPipes(HandlerBase):
|
||||
"""
|
||||
Обработчик получения списка пайп
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
result = []
|
||||
|
||||
for pipe_name, pipe in context.pipe_manager.pipes.items():
|
||||
item = {
|
||||
"name": pipe.name,
|
||||
"description": pipe.description,
|
||||
"input_data_type": ToString(pipe.input_data_type),
|
||||
"output_data_type": ToString(pipe.output_data_type)
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
|
||||
class RunPipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик запуска пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "data"])
|
||||
|
||||
name = parameters["name"]
|
||||
data = parameters["data"]
|
||||
|
||||
pipeline = context.pipeline_manager.get(name)
|
||||
result = pipeline.run(data)
|
||||
|
||||
for subscriber in context.subscribers[name]:
|
||||
try:
|
||||
Send(subscriber.host, subscriber.port, result)
|
||||
|
||||
return result
|
||||
|
||||
class SubscribeOnPipeline(HandlerBase):
|
||||
"""
|
||||
Обработчик подписки на результаты работы пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("subscribe_on_pipeline")
|
||||
|
||||
def do(self, context: Context, parameters: dict):
|
||||
CheckParameters(parameters, ["name", "host", "port"])
|
||||
|
||||
name = parameters["name"]
|
||||
host = parameters["host"]
|
||||
port = parameters["port"]
|
||||
|
||||
context.sub
|
||||
|
||||
return result
|
||||
@@ -1,17 +0,0 @@
|
||||
import json
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class CreatePipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "pipes"])
|
||||
|
||||
name = parameters["name"]
|
||||
pipes = parameters["pipes"]
|
||||
|
||||
context.pipeline_manager.create(name, pipes)
|
||||
|
||||
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
|
||||
context.storage.do(query)
|
||||
@@ -1,18 +0,0 @@
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipelines(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for name, pipeline in context.pipeline_manager.pipelines.items():
|
||||
item = {
|
||||
"name": pipeline.name,
|
||||
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -1,21 +0,0 @@
|
||||
from modules.pipelines.pipes.pipe_base import ToString
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipes(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for pipe_name, pipe in context.pipe_manager.pipes.items():
|
||||
item = {
|
||||
"name": pipe.name,
|
||||
"description": pipe.description,
|
||||
"input_data_type": ToString(pipe.input_data_type),
|
||||
"output_data_type": ToString(pipe.output_data_type)
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -1,16 +0,0 @@
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class RunPipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "data"])
|
||||
|
||||
name = parameters["name"]
|
||||
data = parameters["data"]
|
||||
|
||||
pipeline = context.pipeline_manager.get(name)
|
||||
result = pipeline.run(data)
|
||||
|
||||
return result
|
||||
@@ -6,13 +6,24 @@ class LogLevel(Enum):
|
||||
Debug = 2
|
||||
|
||||
def __int__(self):
|
||||
return self.value
|
||||
return self.value[0]
|
||||
|
||||
def __str__(self):
|
||||
if self.value == LogLevel.Error:
|
||||
return "Error"
|
||||
elif self.value == LogLevel.Warning:
|
||||
return "Warning"
|
||||
elif self.value == LogLevel.Debug:
|
||||
return "Debug"
|
||||
else:
|
||||
return "Unknown log level"
|
||||
|
||||
class Logger:
|
||||
def __init__(self, max_log_level: LogLevel, callback):
|
||||
self.max_log_level = max_log_level
|
||||
self.callback = callback
|
||||
|
||||
def log(self, level, message):
|
||||
if self.callback is not None and int(level) <= int(self.max_log_level):
|
||||
def log(self, level: LogLevel, message):
|
||||
# if self.callback is not None and int(level) <= int(self.max_log_level):
|
||||
if self.callback:
|
||||
self.callback(level, message)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import signal
|
||||
import kafka
|
||||
import kafka.errors
|
||||
import multiprocessing.pool
|
||||
|
||||
class Config:
|
||||
def __init__(self, topic, server, pool_size, pool_cache_limit):
|
||||
self.topic = topic
|
||||
self.server = server
|
||||
self.pool_size = pool_size
|
||||
self.pool_cache_limit = pool_cache_limit
|
||||
|
||||
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
|
||||
def getPoolCacheSize(self):
|
||||
return len(self._cache)
|
||||
|
||||
class Consumer:
|
||||
def __init__(self, config: Config, callback):
|
||||
self.callback = callback
|
||||
self.consumer = kafka.KafkaConsumer(
|
||||
config.topic,
|
||||
bootstrap_servers=[config.server],
|
||||
)
|
||||
|
||||
self.should_stop = False
|
||||
self.pool_cache_limit = config.pool_cache_limit
|
||||
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
|
||||
signal.signal(signal.SIGTERM, self.stop)
|
||||
|
||||
def handlePoolCacheExcess(self):
|
||||
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
|
||||
pass
|
||||
|
||||
def run(self, context):
|
||||
print("consumer is running")
|
||||
|
||||
while not self.should_stop:
|
||||
for message in self.consumer:
|
||||
if self.should_stop:
|
||||
break
|
||||
|
||||
self.callback(context, message.value.decode())
|
||||
|
||||
def stop(self):
|
||||
self.should_stop = True
|
||||
@@ -0,0 +1,14 @@
|
||||
import kafka
|
||||
|
||||
class Config:
|
||||
def __init__(self, server):
|
||||
self.server = server
|
||||
|
||||
class Producer:
|
||||
def __init__(self, config):
|
||||
self.producer = kafka.KafkaProducer(
|
||||
bootstrap_servers=[config.server]
|
||||
)
|
||||
|
||||
def send(self, topic: str, message: str):
|
||||
self.producer.send(topic, message.encode())
|
||||
@@ -20,4 +20,7 @@ class Pipeline:
|
||||
for pipe in self.pipes:
|
||||
data = pipe.do(data)
|
||||
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
@@ -19,6 +19,22 @@ class ToLowerCase(PipeBase):
|
||||
def do(self, data):
|
||||
return data.lower()
|
||||
|
||||
class Strip(PipeBase):
|
||||
"""
|
||||
Удаляет пробелы в начале и конце строки
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("strip", DataType.String, DataType.String)
|
||||
self.description = "removes any leading and trailing whitespaces"
|
||||
|
||||
def do(self, data: str):
|
||||
return data.strip()
|
||||
|
||||
class Split(PipeBase):
|
||||
"""
|
||||
Разбивает строку на токены через разделитель
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
import psycopg2 as pg
|
||||
|
||||
class ConnectionData:
|
||||
def __init__(self):
|
||||
self.host = None
|
||||
self.port = None
|
||||
self.database = None
|
||||
self.user = None
|
||||
self.password = None
|
||||
|
||||
class Storage:
|
||||
def __init__(self, host, port, database, user, password):
|
||||
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
|
||||
def __init__(self, connection_data):
|
||||
self.connection = pg.connect(host=connection_data.host, port=connection_data.port, dbname=connection_data.database, user=connection_data.user, password=connection_data.password)
|
||||
|
||||
def do(self, query, should_return=False):
|
||||
transaction = self.connection.cursor()
|
||||
@@ -12,4 +20,3 @@ class Storage:
|
||||
|
||||
if should_return:
|
||||
return transaction.fetchall()
|
||||
|
||||
|
||||
@@ -9,10 +9,6 @@ class Server:
|
||||
self.callback = callback
|
||||
self.context = None
|
||||
self.server = None
|
||||
self.logger = Logger(LogLevel.Error, None)
|
||||
|
||||
def setLogger(self, max_log_level, callback):
|
||||
self.logger = Logger(max_log_level, callback)
|
||||
|
||||
async def run(self):
|
||||
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
|
||||
@@ -42,11 +38,9 @@ class Server:
|
||||
|
||||
request = request.decode()
|
||||
|
||||
# Вызываем коллбэк
|
||||
response = self.callback(self.context, request)
|
||||
response = struct.pack(">I", len(response)) + response.encode()
|
||||
|
||||
# Отправляем ответ
|
||||
writer.write(response)
|
||||
|
||||
def stop(self):
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
class NgramInfo:
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.frequency = 1
|
||||
|
||||
class NgramsDictionary:
|
||||
def __init__(self, ngram_size):
|
||||
self.ngrams = {}
|
||||
self.ngram_size = ngram_size
|
||||
self.last_id = 0
|
||||
|
||||
def update(self, words):
|
||||
for last_word_index in range(self.ngram_size, len(words) + 1):
|
||||
ngram = ""
|
||||
for current_word_index in range(0, self.ngram_size):
|
||||
ngram += words[last_word_index - self.ngram_size + current_word_index] + " "
|
||||
|
||||
self.addNgram(ngram[:-1])
|
||||
|
||||
def addNgram(self, ngram):
|
||||
if ngram in self.ngrams:
|
||||
self.ngrams[ngram].frequency += 1
|
||||
else:
|
||||
word_info = NgramInfo(self.last_id)
|
||||
self.last_id += 1
|
||||
self.ngrams[ngram] = word_info
|
||||
@@ -0,0 +1,20 @@
|
||||
class Constants:
|
||||
whitespace = r"\s*"
|
||||
unknown_begin_part = r"<(\d{1,2})>"
|
||||
month = r"([A-Za-z]{3})"
|
||||
number = r"(\d+)"
|
||||
time = r"(\d{2}:\d{2}:\d{2})"
|
||||
word = r"(\S+)"
|
||||
anything = r"(.*)"
|
||||
|
||||
class Pattern:
|
||||
def __int__(self):
|
||||
self.parts = []
|
||||
|
||||
def __str__(self):
|
||||
result = ""
|
||||
|
||||
for part in self.parts:
|
||||
result += part[0] + Constants.whitespace
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user