Добавил модули
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
def CheckParameters(parameters, required_keys):
|
||||
for key in required_keys:
|
||||
if key not in parameters:
|
||||
raise Exception(f"there is no key '{key}' in parameters")
|
||||
|
||||
class HandlerBase:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def do(self, context, parameters):
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
class HandlerManager:
|
||||
def __init__(self):
|
||||
self.handlers = {}
|
||||
|
||||
def add(self, handler):
|
||||
name = handler.name
|
||||
if self.contains(name):
|
||||
raise Exception("Handler with name '" + name + "' already exists")
|
||||
|
||||
self.handlers[name] = handler
|
||||
|
||||
def get(self, name):
|
||||
if not self.contains(name):
|
||||
raise Exception("There is no handler with name '" + name + "'")
|
||||
|
||||
return self.handlers[name]
|
||||
|
||||
def contains(self, name):
|
||||
return name in self.handlers
|
||||
@@ -0,0 +1,17 @@
|
||||
import json
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class CreatePipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("create_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "pipes"])
|
||||
|
||||
name = parameters["name"]
|
||||
pipes = parameters["pipes"]
|
||||
|
||||
context.pipeline_manager.create(name, pipes)
|
||||
|
||||
query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');"
|
||||
context.storage.do(query)
|
||||
@@ -0,0 +1,18 @@
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipelines(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipelines")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for name, pipeline in context.pipeline_manager.pipelines.items():
|
||||
item = {
|
||||
"name": pipeline.name,
|
||||
"pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"]
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,21 @@
|
||||
from modules.pipelines.pipes.pipe_base import ToString
|
||||
from modules.handlers.base.handler_base import HandlerBase
|
||||
|
||||
class GetPipes(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("get_pipes")
|
||||
|
||||
def do(self, context, parameters):
|
||||
result = []
|
||||
|
||||
for pipe_name, pipe in context.pipe_manager.pipes.items():
|
||||
item = {
|
||||
"name": pipe.name,
|
||||
"description": pipe.description,
|
||||
"input_data_type": ToString(pipe.input_data_type),
|
||||
"output_data_type": ToString(pipe.output_data_type)
|
||||
}
|
||||
|
||||
result.append(item)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,16 @@
|
||||
from modules.handlers.base.handler_base import *
|
||||
|
||||
class RunPipeline(HandlerBase):
|
||||
def __init__(self):
|
||||
super().__init__("run_pipeline")
|
||||
|
||||
def do(self, context, parameters):
|
||||
CheckParameters(parameters, ["name", "data"])
|
||||
|
||||
name = parameters["name"]
|
||||
data = parameters["data"]
|
||||
|
||||
pipeline = context.pipeline_manager.get(name)
|
||||
result = pipeline.run(data)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,19 @@
|
||||
class PipeManager:
|
||||
def __init__(self):
|
||||
self.pipes = {}
|
||||
|
||||
def add(self, pipe):
|
||||
name = pipe.name
|
||||
if self.contains(name):
|
||||
raise Exception("Pipe with name '" + name + "' already exists")
|
||||
|
||||
self.pipes[name] = pipe
|
||||
|
||||
def get(self, name):
|
||||
if not self.contains(name):
|
||||
raise Exception("There is no pipe with name '" + name + "'")
|
||||
|
||||
return self.pipes[name]
|
||||
|
||||
def contains(self, name):
|
||||
return name in self.pipes
|
||||
@@ -0,0 +1,23 @@
|
||||
from modules.pipelines.pipes.starter import Starter
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.pipes = [Starter()]
|
||||
|
||||
def add(self, pipe):
|
||||
previous_output_data_type = self.pipes[-1].output_data_type
|
||||
current_input_data_type = pipe.input_data_type
|
||||
|
||||
if previous_output_data_type != current_input_data_type:
|
||||
raise Exception("Output and input datatypes are different")
|
||||
|
||||
self.pipes.append(pipe)
|
||||
|
||||
return self
|
||||
|
||||
def run(self, data):
|
||||
for pipe in self.pipes:
|
||||
data = pipe.do(data)
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,25 @@
|
||||
from modules.pipelines.pipeline import Pipeline
|
||||
|
||||
class PipelineManager:
|
||||
def __init__(self, pipe_manager):
|
||||
self.pipe_manager = pipe_manager
|
||||
self.pipelines = {}
|
||||
|
||||
def create(self, name, pipe_names):
|
||||
if self.contains(name):
|
||||
raise Exception("Pipeline with name '" + name + "' already exists")
|
||||
|
||||
pipeline = Pipeline(name)
|
||||
for pipe_name in pipe_names:
|
||||
pipeline.add(self.pipe_manager.get(pipe_name))
|
||||
|
||||
self.pipelines[name] = pipeline
|
||||
|
||||
def get(self, name):
|
||||
if not self.contains(name):
|
||||
raise Exception("There is no pipelines with name '" + name + "'")
|
||||
|
||||
return self.pipelines[name]
|
||||
|
||||
def contains(self, name):
|
||||
return name in self.pipelines
|
||||
@@ -0,0 +1,49 @@
|
||||
import numpy
|
||||
import pandas
|
||||
from collections import defaultdict
|
||||
|
||||
def BuildBigramDictionary(corpus: list):
|
||||
bigrams = []
|
||||
vocabulary = []
|
||||
count_matrix_dictionary = defaultdict(dict)
|
||||
|
||||
for i in range(len(corpus) - 3 + 1):
|
||||
trigram = tuple(corpus[i:i + 3])
|
||||
|
||||
bigram = trigram[0:-1]
|
||||
if bigram not in bigrams:
|
||||
bigrams.append(bigram)
|
||||
|
||||
last_word = trigram[-1]
|
||||
if last_word not in vocabulary:
|
||||
vocabulary.append(last_word)
|
||||
|
||||
if (bigram, last_word) not in count_matrix_dictionary:
|
||||
count_matrix_dictionary[bigram, last_word] = 0
|
||||
|
||||
count_matrix_dictionary[bigram, last_word] += 1
|
||||
|
||||
count_matrix = numpy.zeros((len(bigrams), len(vocabulary)))
|
||||
for trigram_key, trigram_count in count_matrix_dictionary.items():
|
||||
count_matrix[bigrams.index(trigram_key[0]), vocabulary.index(trigram_key[1])] = trigram_count
|
||||
|
||||
count_matrix = pandas.DataFrame(count_matrix, index=bigrams, columns=vocabulary)
|
||||
return count_matrix
|
||||
|
||||
class BigramDictionary:
|
||||
def __init__(self, corpus: list):
|
||||
self.count_matrix = BuildBigramDictionary(corpus)
|
||||
self.probability_matrix = self.count_matrix.div(self.count_matrix.sum(axis="columns"), axis="rows")
|
||||
|
||||
def score(self, line: list) -> float:
|
||||
result = 0.0
|
||||
|
||||
for i in range(len(line) - 3 + 1):
|
||||
trigram = tuple(line[i:i + 3])
|
||||
|
||||
bigram = trigram[:-1]
|
||||
last_word = trigram[-1]
|
||||
if last_word in self.probability_matrix and bigram in self.probability_matrix[last_word]:
|
||||
result += self.probability_matrix[last_word][bigram]
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,16 @@
|
||||
from modules.pipelines.pipes.pipe_base import *
|
||||
from modules.transports.client import Client
|
||||
|
||||
class Connector(PipeBase):
|
||||
def __init__(self, name, host, port):
|
||||
super().__init__(name, DataType.String, DataType.String)
|
||||
self.host = host
|
||||
self.port = port
|
||||
|
||||
def do(self, data):
|
||||
client = Client()
|
||||
client.connect(self.host, self.port)
|
||||
|
||||
client.send(data)
|
||||
|
||||
return client.receive()
|
||||
@@ -0,0 +1,9 @@
|
||||
from modules.pipelines.pipes.pipe_base import *
|
||||
|
||||
class Custom(PipeBase):
|
||||
def __init__(self, name, input_data_type, output_data_type, callback):
|
||||
super().__init__(name, input_data_type, output_data_type)
|
||||
self.callback = callback
|
||||
|
||||
def do(self, data):
|
||||
return self.callback(data)
|
||||
@@ -0,0 +1,54 @@
|
||||
from enum import Enum
|
||||
|
||||
class DataType(Enum):
|
||||
"""
|
||||
Перечисление типов данных пайпы
|
||||
"""
|
||||
|
||||
String = 0, # Строка
|
||||
StringArray = 1, # Массив строк
|
||||
IntegerArray = 2 # Массив целых чисел
|
||||
|
||||
def __cmp__(self, other):
|
||||
return self.value == other.value
|
||||
|
||||
def ToString(data_type) -> str:
|
||||
"""
|
||||
Преобразование типа данных к строковому представлению
|
||||
:param data_type: Тип данных
|
||||
:return: Строка
|
||||
"""
|
||||
|
||||
if data_type == DataType.String:
|
||||
return "string"
|
||||
elif data_type == DataType.StringArray:
|
||||
return "string array"
|
||||
elif data_type == DataType.IntegerArray:
|
||||
return "integer array"
|
||||
|
||||
class PipeBase:
|
||||
"""
|
||||
Базовый класс пайпы
|
||||
"""
|
||||
|
||||
def __init__(self, name, input_data_type, output_data_type):
|
||||
"""
|
||||
Конструктор
|
||||
:param name: Название пайпы
|
||||
:param input_data_type: Тип входных данных
|
||||
:param output_data_type: Тип выходных данных
|
||||
"""
|
||||
|
||||
self.name = name
|
||||
self.description = ""
|
||||
self.input_data_type = input_data_type
|
||||
self.output_data_type = output_data_type
|
||||
|
||||
def do(self, data):
|
||||
"""
|
||||
Вызов обработки данных
|
||||
:param data: Данные
|
||||
:return: Обработанные данные
|
||||
"""
|
||||
|
||||
raise Exception("calling unimplemented method")
|
||||
@@ -0,0 +1,115 @@
|
||||
import string
|
||||
import nltk
|
||||
|
||||
from modules.pipelines.pipes.pipe_base import *
|
||||
|
||||
class ToLowerCase(PipeBase):
|
||||
"""
|
||||
Преобразует все символы в строке к нижнему регистру
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("to_lower_case", DataType.String, DataType.String)
|
||||
self.description = "converts all letters to lower case"
|
||||
|
||||
def do(self, data):
|
||||
return data.lower()
|
||||
|
||||
class Split(PipeBase):
|
||||
"""
|
||||
Разбивает строку на токены через разделитель
|
||||
"""
|
||||
|
||||
def __init__(self, name, divider):
|
||||
"""
|
||||
Конструктор
|
||||
:param name: Название пайпа
|
||||
:param divider: Разделитель
|
||||
"""
|
||||
|
||||
super().__init__(name, DataType.String, DataType.StringArray)
|
||||
self.description = "split string into tokens array by divider"
|
||||
self.divider = divider
|
||||
|
||||
def do(self, data: str):
|
||||
return data.split(self.divider)
|
||||
|
||||
class SplitBySpace(Split):
|
||||
"""
|
||||
Разбивает строку на токены через пробелы
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("split_by_space", " ")
|
||||
self.description = "split string into tokens array by space"
|
||||
|
||||
class Filter(PipeBase):
|
||||
"""
|
||||
Фильтр, базовый класс. Удаляет токены, указанные в конструкторе
|
||||
"""
|
||||
|
||||
def __init__(self, name, tokens_to_remove):
|
||||
"""
|
||||
Конструктор
|
||||
:param name: Название фильтра
|
||||
:param tokens_to_remove: Удаляемые токены
|
||||
"""
|
||||
|
||||
super().__init__(name, DataType.StringArray, DataType.StringArray)
|
||||
self.tokens_to_remove = tokens_to_remove
|
||||
|
||||
def do(self, data):
|
||||
data = [token for token in data if not token in self.tokens_to_remove]
|
||||
return data
|
||||
|
||||
class RemovePunctuationMarks(Filter):
|
||||
"""
|
||||
Фильтр. Удаляет знаки пунктуации
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("remove_punctuation_marks", [c for c in string.punctuation])
|
||||
self.description = "removes punctuation marks"
|
||||
|
||||
class RemoveStopWords(Filter):
|
||||
"""
|
||||
Фильтр. Удаляет стоп-слова
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("remove_stop_words", set(nltk.corpus.stopwords.words("english")))
|
||||
self.description = "removes stop words, such as 'the', 'is' and etc"
|
||||
|
||||
class Stem(PipeBase):
|
||||
"""
|
||||
Оставляет от каждого слова только корень
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("stem", DataType.StringArray, DataType.StringArray)
|
||||
self.description = "stem all words, like 'loving' become 'love'"
|
||||
|
||||
def do(self, data):
|
||||
porter = nltk.stem.porter.PorterStemmer()
|
||||
data = [porter.stem(word) for word in data]
|
||||
return data
|
||||
@@ -0,0 +1,16 @@
|
||||
from modules.pipelines.pipes.pipe_base import *
|
||||
|
||||
class Starter(PipeBase):
|
||||
"""
|
||||
Класс-стартер для запуска пайплайна
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Конструктор
|
||||
"""
|
||||
|
||||
super().__init__("_starter", DataType.String, DataType.String)
|
||||
|
||||
def do(self, data):
|
||||
return data
|
||||
@@ -0,0 +1,15 @@
|
||||
import psycopg2 as pg
|
||||
|
||||
class Storage:
|
||||
def __init__(self, host, port, database, user, password):
|
||||
self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password)
|
||||
|
||||
def do(self, query, should_return=False):
|
||||
transaction = self.connection.cursor()
|
||||
transaction.execute(query)
|
||||
|
||||
self.connection.commit()
|
||||
|
||||
if should_return:
|
||||
return transaction.fetchall()
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import socket
|
||||
|
||||
class Client:
|
||||
def __init__(self):
|
||||
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
|
||||
def connect(self, host, port):
|
||||
self.socket.connect((host, port))
|
||||
|
||||
def disconnect(self):
|
||||
self.socket.close()
|
||||
|
||||
def send(self, data):
|
||||
self.socket.sendall(data.encode())
|
||||
|
||||
def receive(self):
|
||||
part_size = 4096
|
||||
buffer = b""
|
||||
|
||||
while True:
|
||||
part = self.socket.recv(part_size)
|
||||
buffer += part
|
||||
|
||||
if len(part) < part_size:
|
||||
break
|
||||
|
||||
return buffer.decode()
|
||||
@@ -0,0 +1,37 @@
|
||||
import asyncio
|
||||
|
||||
class Server:
|
||||
def __init__(self, host, port, callback):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.callback = callback
|
||||
self.context = None
|
||||
self.server = None
|
||||
|
||||
async def run(self):
|
||||
self.server = await asyncio.start_server(self.handleClient, self.host, self.port)
|
||||
print("server is running")
|
||||
|
||||
async with self.server:
|
||||
await self.server.serve_forever()
|
||||
|
||||
async def handleClient(self, reader, writer):
|
||||
print("accepted new client")
|
||||
|
||||
part_size = 4096
|
||||
request = b""
|
||||
|
||||
while True:
|
||||
part = await asyncio.wait_for(reader.read(part_size), timeout=5.0)
|
||||
request += part
|
||||
|
||||
if len(part) < part_size:
|
||||
break
|
||||
|
||||
request = request.decode()
|
||||
|
||||
response = self.callback(self.context, request)
|
||||
writer.write(response.encode())
|
||||
|
||||
def stop(self):
|
||||
self.server.stop()
|
||||
Reference in New Issue
Block a user