From 1d365127568912582948080283e72c3dd1d722db Mon Sep 17 00:00:00 2001 From: Kirill Volokh Date: Tue, 5 Dec 2023 00:10:42 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=BC=D0=BE=D0=B4=D1=83=D0=BB=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/handlers/base/handler_base.py | 11 ++ modules/handlers/handler_manager.py | 19 +++ modules/handlers/pipelines/create_pipeline.py | 17 +++ modules/handlers/pipelines/get_pipelines.py | 18 +++ modules/handlers/pipelines/get_pipes.py | 21 ++++ modules/handlers/pipelines/run_pipeline.py | 16 +++ modules/pipelines/pipe_manager.py | 19 +++ modules/pipelines/pipeline.py | 23 ++++ modules/pipelines/pipeline_manager.py | 25 ++++ modules/pipelines/pipes/bigram_dictionary.py | 49 ++++++++ modules/pipelines/pipes/connector.py | 16 +++ modules/pipelines/pipes/custom.py | 9 ++ modules/pipelines/pipes/pipe_base.py | 54 ++++++++ modules/pipelines/pipes/preprocessing.py | 115 ++++++++++++++++++ modules/pipelines/pipes/starter.py | 16 +++ modules/storages/storage.py | 15 +++ modules/transports/client.py | 27 ++++ modules/transports/server.py | 37 ++++++ 18 files changed, 507 insertions(+) create mode 100644 modules/handlers/base/handler_base.py create mode 100644 modules/handlers/handler_manager.py create mode 100644 modules/handlers/pipelines/create_pipeline.py create mode 100644 modules/handlers/pipelines/get_pipelines.py create mode 100644 modules/handlers/pipelines/get_pipes.py create mode 100644 modules/handlers/pipelines/run_pipeline.py create mode 100644 modules/pipelines/pipe_manager.py create mode 100644 modules/pipelines/pipeline.py create mode 100644 modules/pipelines/pipeline_manager.py create mode 100644 modules/pipelines/pipes/bigram_dictionary.py create mode 100644 modules/pipelines/pipes/connector.py create mode 100644 modules/pipelines/pipes/custom.py create mode 100644 modules/pipelines/pipes/pipe_base.py create mode 100644 modules/pipelines/pipes/preprocessing.py create mode 100644 modules/pipelines/pipes/starter.py create mode 100644 modules/storages/storage.py create mode 100644 modules/transports/client.py create mode 100644 modules/transports/server.py diff --git a/modules/handlers/base/handler_base.py b/modules/handlers/base/handler_base.py new file mode 100644 index 0000000..e42c0e1 --- /dev/null +++ b/modules/handlers/base/handler_base.py @@ -0,0 +1,11 @@ +def CheckParameters(parameters, required_keys): + for key in required_keys: + if key not in parameters: + raise Exception(f"there is no key '{key}' in parameters") + +class HandlerBase: + def __init__(self, name): + self.name = name + + def do(self, context, parameters): + pass diff --git a/modules/handlers/handler_manager.py b/modules/handlers/handler_manager.py new file mode 100644 index 0000000..e02236c --- /dev/null +++ b/modules/handlers/handler_manager.py @@ -0,0 +1,19 @@ +class HandlerManager: + def __init__(self): + self.handlers = {} + + def add(self, handler): + name = handler.name + if self.contains(name): + raise Exception("Handler with name '" + name + "' already exists") + + self.handlers[name] = handler + + def get(self, name): + if not self.contains(name): + raise Exception("There is no handler with name '" + name + "'") + + return self.handlers[name] + + def contains(self, name): + return name in self.handlers \ No newline at end of file diff --git a/modules/handlers/pipelines/create_pipeline.py b/modules/handlers/pipelines/create_pipeline.py new file mode 100644 index 0000000..d43ae4a --- /dev/null +++ b/modules/handlers/pipelines/create_pipeline.py @@ -0,0 +1,17 @@ +import json +from modules.handlers.base.handler_base import * + +class CreatePipeline(HandlerBase): + def __init__(self): + super().__init__("create_pipeline") + + def do(self, context, parameters): + CheckParameters(parameters, ["name", "pipes"]) + + name = parameters["name"] + pipes = parameters["pipes"] + + context.pipeline_manager.create(name, pipes) + + query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');" + context.storage.do(query) diff --git a/modules/handlers/pipelines/get_pipelines.py b/modules/handlers/pipelines/get_pipelines.py new file mode 100644 index 0000000..49e0e4e --- /dev/null +++ b/modules/handlers/pipelines/get_pipelines.py @@ -0,0 +1,18 @@ +from modules.handlers.base.handler_base import HandlerBase + +class GetPipelines(HandlerBase): + def __init__(self): + super().__init__("get_pipelines") + + def do(self, context, parameters): + result = [] + + for name, pipeline in context.pipeline_manager.pipelines.items(): + item = { + "name": pipeline.name, + "pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"] + } + + result.append(item) + + return result diff --git a/modules/handlers/pipelines/get_pipes.py b/modules/handlers/pipelines/get_pipes.py new file mode 100644 index 0000000..9ef8701 --- /dev/null +++ b/modules/handlers/pipelines/get_pipes.py @@ -0,0 +1,21 @@ +from modules.pipelines.pipes.pipe_base import ToString +from modules.handlers.base.handler_base import HandlerBase + +class GetPipes(HandlerBase): + def __init__(self): + super().__init__("get_pipes") + + def do(self, context, parameters): + result = [] + + for pipe_name, pipe in context.pipe_manager.pipes.items(): + item = { + "name": pipe.name, + "description": pipe.description, + "input_data_type": ToString(pipe.input_data_type), + "output_data_type": ToString(pipe.output_data_type) + } + + result.append(item) + + return result diff --git a/modules/handlers/pipelines/run_pipeline.py b/modules/handlers/pipelines/run_pipeline.py new file mode 100644 index 0000000..9ff5559 --- /dev/null +++ b/modules/handlers/pipelines/run_pipeline.py @@ -0,0 +1,16 @@ +from modules.handlers.base.handler_base import * + +class RunPipeline(HandlerBase): + def __init__(self): + super().__init__("run_pipeline") + + def do(self, context, parameters): + CheckParameters(parameters, ["name", "data"]) + + name = parameters["name"] + data = parameters["data"] + + pipeline = context.pipeline_manager.get(name) + result = pipeline.run(data) + + return result diff --git a/modules/pipelines/pipe_manager.py b/modules/pipelines/pipe_manager.py new file mode 100644 index 0000000..16f811a --- /dev/null +++ b/modules/pipelines/pipe_manager.py @@ -0,0 +1,19 @@ +class PipeManager: + def __init__(self): + self.pipes = {} + + def add(self, pipe): + name = pipe.name + if self.contains(name): + raise Exception("Pipe with name '" + name + "' already exists") + + self.pipes[name] = pipe + + def get(self, name): + if not self.contains(name): + raise Exception("There is no pipe with name '" + name + "'") + + return self.pipes[name] + + def contains(self, name): + return name in self.pipes diff --git a/modules/pipelines/pipeline.py b/modules/pipelines/pipeline.py new file mode 100644 index 0000000..3d14f22 --- /dev/null +++ b/modules/pipelines/pipeline.py @@ -0,0 +1,23 @@ +from modules.pipelines.pipes.starter import Starter + +class Pipeline: + def __init__(self, name): + self.name = name + self.pipes = [Starter()] + + def add(self, pipe): + previous_output_data_type = self.pipes[-1].output_data_type + current_input_data_type = pipe.input_data_type + + if previous_output_data_type != current_input_data_type: + raise Exception("Output and input datatypes are different") + + self.pipes.append(pipe) + + return self + + def run(self, data): + for pipe in self.pipes: + data = pipe.do(data) + + return data diff --git a/modules/pipelines/pipeline_manager.py b/modules/pipelines/pipeline_manager.py new file mode 100644 index 0000000..35bbb34 --- /dev/null +++ b/modules/pipelines/pipeline_manager.py @@ -0,0 +1,25 @@ +from modules.pipelines.pipeline import Pipeline + +class PipelineManager: + def __init__(self, pipe_manager): + self.pipe_manager = pipe_manager + self.pipelines = {} + + def create(self, name, pipe_names): + if self.contains(name): + raise Exception("Pipeline with name '" + name + "' already exists") + + pipeline = Pipeline(name) + for pipe_name in pipe_names: + pipeline.add(self.pipe_manager.get(pipe_name)) + + self.pipelines[name] = pipeline + + def get(self, name): + if not self.contains(name): + raise Exception("There is no pipelines with name '" + name + "'") + + return self.pipelines[name] + + def contains(self, name): + return name in self.pipelines diff --git a/modules/pipelines/pipes/bigram_dictionary.py b/modules/pipelines/pipes/bigram_dictionary.py new file mode 100644 index 0000000..c41a74b --- /dev/null +++ b/modules/pipelines/pipes/bigram_dictionary.py @@ -0,0 +1,49 @@ +import numpy +import pandas +from collections import defaultdict + +def BuildBigramDictionary(corpus: list): + bigrams = [] + vocabulary = [] + count_matrix_dictionary = defaultdict(dict) + + for i in range(len(corpus) - 3 + 1): + trigram = tuple(corpus[i:i + 3]) + + bigram = trigram[0:-1] + if bigram not in bigrams: + bigrams.append(bigram) + + last_word = trigram[-1] + if last_word not in vocabulary: + vocabulary.append(last_word) + + if (bigram, last_word) not in count_matrix_dictionary: + count_matrix_dictionary[bigram, last_word] = 0 + + count_matrix_dictionary[bigram, last_word] += 1 + + count_matrix = numpy.zeros((len(bigrams), len(vocabulary))) + for trigram_key, trigram_count in count_matrix_dictionary.items(): + count_matrix[bigrams.index(trigram_key[0]), vocabulary.index(trigram_key[1])] = trigram_count + + count_matrix = pandas.DataFrame(count_matrix, index=bigrams, columns=vocabulary) + return count_matrix + +class BigramDictionary: + def __init__(self, corpus: list): + self.count_matrix = BuildBigramDictionary(corpus) + self.probability_matrix = self.count_matrix.div(self.count_matrix.sum(axis="columns"), axis="rows") + + def score(self, line: list) -> float: + result = 0.0 + + for i in range(len(line) - 3 + 1): + trigram = tuple(line[i:i + 3]) + + bigram = trigram[:-1] + last_word = trigram[-1] + if last_word in self.probability_matrix and bigram in self.probability_matrix[last_word]: + result += self.probability_matrix[last_word][bigram] + + return result diff --git a/modules/pipelines/pipes/connector.py b/modules/pipelines/pipes/connector.py new file mode 100644 index 0000000..0902d3a --- /dev/null +++ b/modules/pipelines/pipes/connector.py @@ -0,0 +1,16 @@ +from modules.pipelines.pipes.pipe_base import * +from modules.transports.client import Client + +class Connector(PipeBase): + def __init__(self, name, host, port): + super().__init__(name, DataType.String, DataType.String) + self.host = host + self.port = port + + def do(self, data): + client = Client() + client.connect(self.host, self.port) + + client.send(data) + + return client.receive() diff --git a/modules/pipelines/pipes/custom.py b/modules/pipelines/pipes/custom.py new file mode 100644 index 0000000..9e8a0da --- /dev/null +++ b/modules/pipelines/pipes/custom.py @@ -0,0 +1,9 @@ +from modules.pipelines.pipes.pipe_base import * + +class Custom(PipeBase): + def __init__(self, name, input_data_type, output_data_type, callback): + super().__init__(name, input_data_type, output_data_type) + self.callback = callback + + def do(self, data): + return self.callback(data) diff --git a/modules/pipelines/pipes/pipe_base.py b/modules/pipelines/pipes/pipe_base.py new file mode 100644 index 0000000..769edaf --- /dev/null +++ b/modules/pipelines/pipes/pipe_base.py @@ -0,0 +1,54 @@ +from enum import Enum + +class DataType(Enum): + """ + Перечисление типов данных пайпы + """ + + String = 0, # Строка + StringArray = 1, # Массив строк + IntegerArray = 2 # Массив целых чисел + + def __cmp__(self, other): + return self.value == other.value + +def ToString(data_type) -> str: + """ + Преобразование типа данных к строковому представлению + :param data_type: Тип данных + :return: Строка + """ + + if data_type == DataType.String: + return "string" + elif data_type == DataType.StringArray: + return "string array" + elif data_type == DataType.IntegerArray: + return "integer array" + +class PipeBase: + """ + Базовый класс пайпы + """ + + def __init__(self, name, input_data_type, output_data_type): + """ + Конструктор + :param name: Название пайпы + :param input_data_type: Тип входных данных + :param output_data_type: Тип выходных данных + """ + + self.name = name + self.description = "" + self.input_data_type = input_data_type + self.output_data_type = output_data_type + + def do(self, data): + """ + Вызов обработки данных + :param data: Данные + :return: Обработанные данные + """ + + raise Exception("calling unimplemented method") diff --git a/modules/pipelines/pipes/preprocessing.py b/modules/pipelines/pipes/preprocessing.py new file mode 100644 index 0000000..e9aca30 --- /dev/null +++ b/modules/pipelines/pipes/preprocessing.py @@ -0,0 +1,115 @@ +import string +import nltk + +from modules.pipelines.pipes.pipe_base import * + +class ToLowerCase(PipeBase): + """ + Преобразует все символы в строке к нижнему регистру + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("to_lower_case", DataType.String, DataType.String) + self.description = "converts all letters to lower case" + + def do(self, data): + return data.lower() + +class Split(PipeBase): + """ + Разбивает строку на токены через разделитель + """ + + def __init__(self, name, divider): + """ + Конструктор + :param name: Название пайпа + :param divider: Разделитель + """ + + super().__init__(name, DataType.String, DataType.StringArray) + self.description = "split string into tokens array by divider" + self.divider = divider + + def do(self, data: str): + return data.split(self.divider) + +class SplitBySpace(Split): + """ + Разбивает строку на токены через пробелы + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("split_by_space", " ") + self.description = "split string into tokens array by space" + +class Filter(PipeBase): + """ + Фильтр, базовый класс. Удаляет токены, указанные в конструкторе + """ + + def __init__(self, name, tokens_to_remove): + """ + Конструктор + :param name: Название фильтра + :param tokens_to_remove: Удаляемые токены + """ + + super().__init__(name, DataType.StringArray, DataType.StringArray) + self.tokens_to_remove = tokens_to_remove + + def do(self, data): + data = [token for token in data if not token in self.tokens_to_remove] + return data + +class RemovePunctuationMarks(Filter): + """ + Фильтр. Удаляет знаки пунктуации + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("remove_punctuation_marks", [c for c in string.punctuation]) + self.description = "removes punctuation marks" + +class RemoveStopWords(Filter): + """ + Фильтр. Удаляет стоп-слова + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("remove_stop_words", set(nltk.corpus.stopwords.words("english"))) + self.description = "removes stop words, such as 'the', 'is' and etc" + +class Stem(PipeBase): + """ + Оставляет от каждого слова только корень + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("stem", DataType.StringArray, DataType.StringArray) + self.description = "stem all words, like 'loving' become 'love'" + + def do(self, data): + porter = nltk.stem.porter.PorterStemmer() + data = [porter.stem(word) for word in data] + return data diff --git a/modules/pipelines/pipes/starter.py b/modules/pipelines/pipes/starter.py new file mode 100644 index 0000000..cc4794e --- /dev/null +++ b/modules/pipelines/pipes/starter.py @@ -0,0 +1,16 @@ +from modules.pipelines.pipes.pipe_base import * + +class Starter(PipeBase): + """ + Класс-стартер для запуска пайплайна + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("_starter", DataType.String, DataType.String) + + def do(self, data): + return data diff --git a/modules/storages/storage.py b/modules/storages/storage.py new file mode 100644 index 0000000..1e0c166 --- /dev/null +++ b/modules/storages/storage.py @@ -0,0 +1,15 @@ +import psycopg2 as pg + +class Storage: + def __init__(self, host, port, database, user, password): + self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password) + + def do(self, query, should_return=False): + transaction = self.connection.cursor() + transaction.execute(query) + + self.connection.commit() + + if should_return: + return transaction.fetchall() + diff --git a/modules/transports/client.py b/modules/transports/client.py new file mode 100644 index 0000000..0b450a2 --- /dev/null +++ b/modules/transports/client.py @@ -0,0 +1,27 @@ +import socket + +class Client: + def __init__(self): + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def connect(self, host, port): + self.socket.connect((host, port)) + + def disconnect(self): + self.socket.close() + + def send(self, data): + self.socket.sendall(data.encode()) + + def receive(self): + part_size = 4096 + buffer = b"" + + while True: + part = self.socket.recv(part_size) + buffer += part + + if len(part) < part_size: + break + + return buffer.decode() diff --git a/modules/transports/server.py b/modules/transports/server.py new file mode 100644 index 0000000..b74043a --- /dev/null +++ b/modules/transports/server.py @@ -0,0 +1,37 @@ +import asyncio + +class Server: + def __init__(self, host, port, callback): + self.host = host + self.port = port + self.callback = callback + self.context = None + self.server = None + + async def run(self): + self.server = await asyncio.start_server(self.handleClient, self.host, self.port) + print("server is running") + + async with self.server: + await self.server.serve_forever() + + async def handleClient(self, reader, writer): + print("accepted new client") + + part_size = 4096 + request = b"" + + while True: + part = await asyncio.wait_for(reader.read(part_size), timeout=5.0) + request += part + + if len(part) < part_size: + break + + request = request.decode() + + response = self.callback(self.context, request) + writer.write(response.encode()) + + def stop(self): + self.server.stop()