Добавил модули

This commit is contained in:
2023-12-05 00:10:42 +03:00
parent 249a9e8a63
commit 1d36512756
18 changed files with 507 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
def CheckParameters(parameters, required_keys):
for key in required_keys:
if key not in parameters:
raise Exception(f"there is no key '{key}' in parameters")
class HandlerBase:
def __init__(self, name):
self.name = name
def do(self, context, parameters):
pass
+19
View File
@@ -0,0 +1,19 @@
class HandlerManager:
def __init__(self):
self.handlers = {}
def add(self, handler):
name = handler.name
if self.contains(name):
raise Exception("Handler with name '" + name + "' already exists")
self.handlers[name] = handler
def get(self, name):
if not self.contains(name):
raise Exception("There is no handler with name '" + name + "'")
return self.handlers[name]
def contains(self, name):
return name in self.handlers
@@ -0,0 +1,17 @@
import json
from modules.handlers.base.handler_base import *
class CreatePipeline(HandlerBase):
def __init__(self):
super().__init__("create_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "pipes"])
name = parameters["name"]
pipes = parameters["pipes"]
context.pipeline_manager.create(name, pipes)
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
context.storage.do(query)
@@ -0,0 +1,18 @@
from modules.handlers.base.handler_base import HandlerBase
class GetPipelines(HandlerBase):
def __init__(self):
super().__init__("get_pipelines")
def do(self, context, parameters):
result = []
for name, pipeline in context.pipeline_manager.pipelines.items():
item = {
"name": pipeline.name,
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
}
result.append(item)
return result
+21
View File
@@ -0,0 +1,21 @@
from modules.pipelines.pipes.pipe_base import ToString
from modules.handlers.base.handler_base import HandlerBase
class GetPipes(HandlerBase):
def __init__(self):
super().__init__("get_pipes")
def do(self, context, parameters):
result = []
for pipe_name, pipe in context.pipe_manager.pipes.items():
item = {
"name": pipe.name,
"description": pipe.description,
"input_data_type": ToString(pipe.input_data_type),
"output_data_type": ToString(pipe.output_data_type)
}
result.append(item)
return result
@@ -0,0 +1,16 @@
from modules.handlers.base.handler_base import *
class RunPipeline(HandlerBase):
def __init__(self):
super().__init__("run_pipeline")
def do(self, context, parameters):
CheckParameters(parameters, ["name", "data"])
name = parameters["name"]
data = parameters["data"]
pipeline = context.pipeline_manager.get(name)
result = pipeline.run(data)
return result
+19
View File
@@ -0,0 +1,19 @@
class PipeManager:
def __init__(self):
self.pipes = {}
def add(self, pipe):
name = pipe.name
if self.contains(name):
raise Exception("Pipe with name '" + name + "' already exists")
self.pipes[name] = pipe
def get(self, name):
if not self.contains(name):
raise Exception("There is no pipe with name '" + name + "'")
return self.pipes[name]
def contains(self, name):
return name in self.pipes
+23
View File
@@ -0,0 +1,23 @@
from modules.pipelines.pipes.starter import Starter
class Pipeline:
def __init__(self, name):
self.name = name
self.pipes = [Starter()]
def add(self, pipe):
previous_output_data_type = self.pipes[-1].output_data_type
current_input_data_type = pipe.input_data_type
if previous_output_data_type != current_input_data_type:
raise Exception("Output and input datatypes are different")
self.pipes.append(pipe)
return self
def run(self, data):
for pipe in self.pipes:
data = pipe.do(data)
return data
+25
View File
@@ -0,0 +1,25 @@
from modules.pipelines.pipeline import Pipeline
class PipelineManager:
def __init__(self, pipe_manager):
self.pipe_manager = pipe_manager
self.pipelines = {}
def create(self, name, pipe_names):
if self.contains(name):
raise Exception("Pipeline with name '" + name + "' already exists")
pipeline = Pipeline(name)
for pipe_name in pipe_names:
pipeline.add(self.pipe_manager.get(pipe_name))
self.pipelines[name] = pipeline
def get(self, name):
if not self.contains(name):
raise Exception("There is no pipelines with name '" + name + "'")
return self.pipelines[name]
def contains(self, name):
return name in self.pipelines
@@ -0,0 +1,49 @@
import numpy
import pandas
from collections import defaultdict
def BuildBigramDictionary(corpus: list):
bigrams = []
vocabulary = []
count_matrix_dictionary = defaultdict(dict)
for i in range(len(corpus) - 3 + 1):
trigram = tuple(corpus[i:i + 3])
bigram = trigram[0:-1]
if bigram not in bigrams:
bigrams.append(bigram)
last_word = trigram[-1]
if last_word not in vocabulary:
vocabulary.append(last_word)
if (bigram, last_word) not in count_matrix_dictionary:
count_matrix_dictionary[bigram, last_word] = 0
count_matrix_dictionary[bigram, last_word] += 1
count_matrix = numpy.zeros((len(bigrams), len(vocabulary)))
for trigram_key, trigram_count in count_matrix_dictionary.items():
count_matrix[bigrams.index(trigram_key[0]), vocabulary.index(trigram_key[1])] = trigram_count
count_matrix = pandas.DataFrame(count_matrix, index=bigrams, columns=vocabulary)
return count_matrix
class BigramDictionary:
def __init__(self, corpus: list):
self.count_matrix = BuildBigramDictionary(corpus)
self.probability_matrix = self.count_matrix.div(self.count_matrix.sum(axis="columns"), axis="rows")
def score(self, line: list) -> float:
result = 0.0
for i in range(len(line) - 3 + 1):
trigram = tuple(line[i:i + 3])
bigram = trigram[:-1]
last_word = trigram[-1]
if last_word in self.probability_matrix and bigram in self.probability_matrix[last_word]:
result += self.probability_matrix[last_word][bigram]
return result
+16
View File
@@ -0,0 +1,16 @@
from modules.pipelines.pipes.pipe_base import *
from modules.transports.client import Client
class Connector(PipeBase):
def __init__(self, name, host, port):
super().__init__(name, DataType.String, DataType.String)
self.host = host
self.port = port
def do(self, data):
client = Client()
client.connect(self.host, self.port)
client.send(data)
return client.receive()
+9
View File
@@ -0,0 +1,9 @@
from modules.pipelines.pipes.pipe_base import *
class Custom(PipeBase):
def __init__(self, name, input_data_type, output_data_type, callback):
super().__init__(name, input_data_type, output_data_type)
self.callback = callback
def do(self, data):
return self.callback(data)
+54
View File
@@ -0,0 +1,54 @@
from enum import Enum
class DataType(Enum):
"""
Перечисление типов данных пайпы
"""
String = 0, # Строка
StringArray = 1, # Массив строк
IntegerArray = 2 # Массив целых чисел
def __cmp__(self, other):
return self.value == other.value
def ToString(data_type) -> str:
"""
Преобразование типа данных к строковому представлению
:param data_type: Тип данных
:return: Строка
"""
if data_type == DataType.String:
return "string"
elif data_type == DataType.StringArray:
return "string array"
elif data_type == DataType.IntegerArray:
return "integer array"
class PipeBase:
"""
Базовый класс пайпы
"""
def __init__(self, name, input_data_type, output_data_type):
"""
Конструктор
:param name: Название пайпы
:param input_data_type: Тип входных данных
:param output_data_type: Тип выходных данных
"""
self.name = name
self.description = ""
self.input_data_type = input_data_type
self.output_data_type = output_data_type
def do(self, data):
"""
Вызов обработки данных
:param data: Данные
:return: Обработанные данные
"""
raise Exception("calling unimplemented method")
+115
View File
@@ -0,0 +1,115 @@
import string
import nltk
from modules.pipelines.pipes.pipe_base import *
class ToLowerCase(PipeBase):
"""
Преобразует все символы в строке к нижнему регистру
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("to_lower_case", DataType.String, DataType.String)
self.description = "converts all letters to lower case"
def do(self, data):
return data.lower()
class Split(PipeBase):
"""
Разбивает строку на токены через разделитель
"""
def __init__(self, name, divider):
"""
Конструктор
:param name: Название пайпа
:param divider: Разделитель
"""
super().__init__(name, DataType.String, DataType.StringArray)
self.description = "split string into tokens array by divider"
self.divider = divider
def do(self, data: str):
return data.split(self.divider)
class SplitBySpace(Split):
"""
Разбивает строку на токены через пробелы
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("split_by_space", " ")
self.description = "split string into tokens array by space"
class Filter(PipeBase):
"""
Фильтр, базовый класс. Удаляет токены, указанные в конструкторе
"""
def __init__(self, name, tokens_to_remove):
"""
Конструктор
:param name: Название фильтра
:param tokens_to_remove: Удаляемые токены
"""
super().__init__(name, DataType.StringArray, DataType.StringArray)
self.tokens_to_remove = tokens_to_remove
def do(self, data):
data = [token for token in data if not token in self.tokens_to_remove]
return data
class RemovePunctuationMarks(Filter):
"""
Фильтр. Удаляет знаки пунктуации
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("remove_punctuation_marks", [c for c in string.punctuation])
self.description = "removes punctuation marks"
class RemoveStopWords(Filter):
"""
Фильтр. Удаляет стоп-слова
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("remove_stop_words", set(nltk.corpus.stopwords.words("english")))
self.description = "removes stop words, such as 'the', 'is' and etc"
class Stem(PipeBase):
"""
Оставляет от каждого слова только корень
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("stem", DataType.StringArray, DataType.StringArray)
self.description = "stem all words, like 'loving' become 'love'"
def do(self, data):
porter = nltk.stem.porter.PorterStemmer()
data = [porter.stem(word) for word in data]
return data
+16
View File
@@ -0,0 +1,16 @@
from modules.pipelines.pipes.pipe_base import *
class Starter(PipeBase):
"""
Класс-стартер для запуска пайплайна
"""
def __init__(self):
"""
Конструктор
"""
super().__init__("_starter", DataType.String, DataType.String)
def do(self, data):
return data
+15
View File
@@ -0,0 +1,15 @@
import psycopg2 as pg
class Storage:
def __init__(self, host, port, database, user, password):
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
def do(self, query, should_return=False):
transaction = self.connection.cursor()
transaction.execute(query)
self.connection.commit()
if should_return:
return transaction.fetchall()
+27
View File
@@ -0,0 +1,27 @@
import socket
class Client:
def __init__(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self, host, port):
self.socket.connect((host, port))
def disconnect(self):
self.socket.close()
def send(self, data):
self.socket.sendall(data.encode())
def receive(self):
part_size = 4096
buffer = b""
while True:
part = self.socket.recv(part_size)
buffer += part
if len(part) < part_size:
break
return buffer.decode()
+37
View File
@@ -0,0 +1,37 @@
import asyncio
class Server:
def __init__(self, host, port, callback):
self.host = host
self.port = port
self.callback = callback
self.context = None
self.server = None
async def run(self):
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
print("server is running")
async with self.server:
await self.server.serve_forever()
async def handleClient(self, reader, writer):
print("accepted new client")
part_size = 4096
request = b""
while True:
part = await asyncio.wait_for(reader.read(part_size), timeout=5.0)
request += part
if len(part) < part_size:
break
request = request.decode()
response = self.callback(self.context, request)
writer.write(response.encode())
def stop(self):
self.server.stop()