From 026cc9b6bbcb382013ada2e5e773e82bf1784f5e Mon Sep 17 00:00:00 2001 From: Kirill Volokh Date: Mon, 15 Jan 2024 03:45:18 +0300 Subject: [PATCH] =?UTF-8?q?=D0=91=D0=BE=D0=BB=D1=8C=D1=88=D0=BE=D0=B5=20?= =?UTF-8?q?=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?=D0=B2=D1=81=D0=B5=D0=B3=D0=BE=20=D0=BF=D1=80=D0=BE=D0=B5=D0=BA?= =?UTF-8?q?=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/docker-compose.yml | 24 +++ modules/handlers/context.py | 18 ++ modules/handlers/{base => }/handler_base.py | 4 +- modules/handlers/pipelines.py | 109 ++++++++++ modules/handlers/pipelines/create_pipeline.py | 17 -- modules/handlers/pipelines/get_pipelines.py | 18 -- modules/handlers/pipelines/get_pipes.py | 21 -- modules/handlers/pipelines/run_pipeline.py | 16 -- modules/logging/logger.py | 17 +- modules/message_broker/consumer.py | 45 ++++ modules/message_broker/producer.py | 14 ++ modules/pipelines/pipeline.py | 3 + modules/pipelines/pipes/preprocessing.py | 16 ++ modules/storages/storage.py | 13 +- modules/transports/server.py | 6 - .../ngrams_dictionary/ngrams_dictionary.py | 26 +++ modules/utils/pattern.py | 20 ++ readme.md | 25 --- services/console_client/commands.py | 193 ------------------ services/console_client/console_client.py | 70 ------- services/control/context.py | 19 -- services/control/control.py | 130 ------------ services/control/create_context.py | 62 ------ services/control/response.py | 26 --- services/dispatchers/all_dispatcher.py | 41 ++++ services/dispatchers/ceph_osd_dispatcher.py | 43 ++++ services/dispatchers/dispatcher_base.py | 51 +++++ services/log_analyzer/__init__.py | 4 + services/log_analyzer/__main__.py | 138 +++++++++++++ services/log_analyzer/model.py | 86 ++++++++ services/rocksdb_analyzer/pipes.py | 33 +++ services/rocksdb_analyzer/rocksdb_analyzer.py | 22 ++ simple_consumer.py | 13 ++ simple_producer.py | 12 ++ sql/storage.sql | 7 - sql/tables.sql | 15 ++ 36 files changed, 760 insertions(+), 617 deletions(-) create mode 100644 docker/docker-compose.yml create mode 100644 modules/handlers/context.py rename modules/handlers/{base => }/handler_base.py (70%) create mode 100644 modules/handlers/pipelines.py delete mode 100644 modules/handlers/pipelines/create_pipeline.py delete mode 100644 modules/handlers/pipelines/get_pipelines.py delete mode 100644 modules/handlers/pipelines/get_pipes.py delete mode 100644 modules/handlers/pipelines/run_pipeline.py create mode 100644 modules/message_broker/consumer.py create mode 100644 modules/message_broker/producer.py create mode 100644 modules/utils/ngrams_dictionary/ngrams_dictionary.py create mode 100644 modules/utils/pattern.py delete mode 100644 services/console_client/commands.py delete mode 100644 services/console_client/console_client.py delete mode 100644 services/control/context.py delete mode 100644 services/control/control.py delete mode 100644 services/control/create_context.py delete mode 100644 services/control/response.py create mode 100644 services/dispatchers/all_dispatcher.py create mode 100644 services/dispatchers/ceph_osd_dispatcher.py create mode 100644 services/dispatchers/dispatcher_base.py create mode 100644 services/log_analyzer/__init__.py create mode 100644 services/log_analyzer/__main__.py create mode 100644 services/log_analyzer/model.py create mode 100644 services/rocksdb_analyzer/pipes.py create mode 100644 services/rocksdb_analyzer/rocksdb_analyzer.py create mode 100644 simple_consumer.py create mode 100644 simple_producer.py delete mode 100644 sql/storage.sql create mode 100644 sql/tables.sql diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..f1ffb99 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,24 @@ +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ports: + - 22181:2181 + + kafka: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper + ports: + - 29092:29092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + diff --git a/modules/handlers/context.py b/modules/handlers/context.py new file mode 100644 index 0000000..841f1a6 --- /dev/null +++ b/modules/handlers/context.py @@ -0,0 +1,18 @@ +class Subscriber: + def __init__(self): + self.host = None + self.port = None + +class Context: + """ + Контекст обработчиков + """ + + def __init__(self): + """ + Конструктор + """ + + self.pipe_manager = None + self.pipeline_manager = None + self.subscribers = {} diff --git a/modules/handlers/base/handler_base.py b/modules/handlers/handler_base.py similarity index 70% rename from modules/handlers/base/handler_base.py rename to modules/handlers/handler_base.py index e42c0e1..d2d24f1 100644 --- a/modules/handlers/base/handler_base.py +++ b/modules/handlers/handler_base.py @@ -1,3 +1,5 @@ +from modules.handlers.context import Context + def CheckParameters(parameters, required_keys): for key in required_keys: if key not in parameters: @@ -7,5 +9,5 @@ class HandlerBase: def __init__(self, name): self.name = name - def do(self, context, parameters): + def do(self, context: Context, parameters: dict): pass diff --git a/modules/handlers/pipelines.py b/modules/handlers/pipelines.py new file mode 100644 index 0000000..67a7b36 --- /dev/null +++ b/modules/handlers/pipelines.py @@ -0,0 +1,109 @@ +import json + +from modules.pipelines.pipes.pipe_base import ToString +from modules.handlers.handler_base import * + +class CreatePipeline(HandlerBase): + """ + Обработчик создания пайплайна + """ + + def __init__(self): + super().__init__("create_pipeline") + + def do(self, context: Context, parameters: dict): + CheckParameters(parameters, ["name", "pipes"]) + + name = parameters["name"] + pipes = parameters["pipes"] + + context.pipeline_manager.create(name, pipes) + + query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');" + context.storage.do(query) + +class GetPipelines(HandlerBase): + """ + Обработчик получения списка пайплайнов + """ + + def __init__(self): + super().__init__("get_pipelines") + + def do(self, context: Context, parameters: dict): + result = [] + + for name, pipeline in context.pipeline_manager.pipelines.items(): + item = { + "name": pipeline.name, + "pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"] + } + + result.append(item) + + return result + +class GetPipes(HandlerBase): + """ + Обработчик получения списка пайп + """ + + def __init__(self): + super().__init__("get_pipes") + + def do(self, context: Context, parameters: dict): + result = [] + + for pipe_name, pipe in context.pipe_manager.pipes.items(): + item = { + "name": pipe.name, + "description": pipe.description, + "input_data_type": ToString(pipe.input_data_type), + "output_data_type": ToString(pipe.output_data_type) + } + + result.append(item) + + return result + +class RunPipeline(HandlerBase): + """ + Обработчик запуска пайплайна + """ + + def __init__(self): + super().__init__("run_pipeline") + + def do(self, context: Context, parameters: dict): + CheckParameters(parameters, ["name", "data"]) + + name = parameters["name"] + data = parameters["data"] + + pipeline = context.pipeline_manager.get(name) + result = pipeline.run(data) + + for subscriber in context.subscribers[name]: + try: + Send(subscriber.host, subscriber.port, result) + + return result + +class SubscribeOnPipeline(HandlerBase): + """ + Обработчик подписки на результаты работы пайплайна + """ + + def __init__(self): + super().__init__("subscribe_on_pipeline") + + def do(self, context: Context, parameters: dict): + CheckParameters(parameters, ["name", "host", "port"]) + + name = parameters["name"] + host = parameters["host"] + port = parameters["port"] + + context.sub + + return result diff --git a/modules/handlers/pipelines/create_pipeline.py b/modules/handlers/pipelines/create_pipeline.py deleted file mode 100644 index d43ae4a..0000000 --- a/modules/handlers/pipelines/create_pipeline.py +++ /dev/null @@ -1,17 +0,0 @@ -import json -from modules.handlers.base.handler_base import * - -class CreatePipeline(HandlerBase): - def __init__(self): - super().__init__("create_pipeline") - - def do(self, context, parameters): - CheckParameters(parameters, ["name", "pipes"]) - - name = parameters["name"] - pipes = parameters["pipes"] - - context.pipeline_manager.create(name, pipes) - - query = f"insert into pipelines (name, pipes) values ('{name}', '{json.dumps(pipes)}');" - context.storage.do(query) diff --git a/modules/handlers/pipelines/get_pipelines.py b/modules/handlers/pipelines/get_pipelines.py deleted file mode 100644 index 49e0e4e..0000000 --- a/modules/handlers/pipelines/get_pipelines.py +++ /dev/null @@ -1,18 +0,0 @@ -from modules.handlers.base.handler_base import HandlerBase - -class GetPipelines(HandlerBase): - def __init__(self): - super().__init__("get_pipelines") - - def do(self, context, parameters): - result = [] - - for name, pipeline in context.pipeline_manager.pipelines.items(): - item = { - "name": pipeline.name, - "pipes": [pipe.name for pipe in pipeline.pipes if pipe.name[0] != "_"] - } - - result.append(item) - - return result diff --git a/modules/handlers/pipelines/get_pipes.py b/modules/handlers/pipelines/get_pipes.py deleted file mode 100644 index 9ef8701..0000000 --- a/modules/handlers/pipelines/get_pipes.py +++ /dev/null @@ -1,21 +0,0 @@ -from modules.pipelines.pipes.pipe_base import ToString -from modules.handlers.base.handler_base import HandlerBase - -class GetPipes(HandlerBase): - def __init__(self): - super().__init__("get_pipes") - - def do(self, context, parameters): - result = [] - - for pipe_name, pipe in context.pipe_manager.pipes.items(): - item = { - "name": pipe.name, - "description": pipe.description, - "input_data_type": ToString(pipe.input_data_type), - "output_data_type": ToString(pipe.output_data_type) - } - - result.append(item) - - return result diff --git a/modules/handlers/pipelines/run_pipeline.py b/modules/handlers/pipelines/run_pipeline.py deleted file mode 100644 index 9ff5559..0000000 --- a/modules/handlers/pipelines/run_pipeline.py +++ /dev/null @@ -1,16 +0,0 @@ -from modules.handlers.base.handler_base import * - -class RunPipeline(HandlerBase): - def __init__(self): - super().__init__("run_pipeline") - - def do(self, context, parameters): - CheckParameters(parameters, ["name", "data"]) - - name = parameters["name"] - data = parameters["data"] - - pipeline = context.pipeline_manager.get(name) - result = pipeline.run(data) - - return result diff --git a/modules/logging/logger.py b/modules/logging/logger.py index ecfbc96..e2cdc01 100644 --- a/modules/logging/logger.py +++ b/modules/logging/logger.py @@ -6,13 +6,24 @@ class LogLevel(Enum): Debug = 2 def __int__(self): - return self.value + return self.value[0] + + def __str__(self): + if self.value == LogLevel.Error: + return "Error" + elif self.value == LogLevel.Warning: + return "Warning" + elif self.value == LogLevel.Debug: + return "Debug" + else: + return "Unknown log level" class Logger: def __init__(self, max_log_level: LogLevel, callback): self.max_log_level = max_log_level self.callback = callback - def log(self, level, message): - if self.callback is not None and int(level) <= int(self.max_log_level): + def log(self, level: LogLevel, message): + # if self.callback is not None and int(level) <= int(self.max_log_level): + if self.callback: self.callback(level, message) diff --git a/modules/message_broker/consumer.py b/modules/message_broker/consumer.py new file mode 100644 index 0000000..5d70526 --- /dev/null +++ b/modules/message_broker/consumer.py @@ -0,0 +1,45 @@ +import signal +import kafka +import kafka.errors +import multiprocessing.pool + +class Config: + def __init__(self, topic, server, pool_size, pool_cache_limit): + self.topic = topic + self.server = server + self.pool_size = pool_size + self.pool_cache_limit = pool_cache_limit + +class LimitedMultiprocessingPool(multiprocessing.pool.Pool): + def getPoolCacheSize(self): + return len(self._cache) + +class Consumer: + def __init__(self, config: Config, callback): + self.callback = callback + self.consumer = kafka.KafkaConsumer( + config.topic, + bootstrap_servers=[config.server], + ) + + self.should_stop = False + self.pool_cache_limit = config.pool_cache_limit + self.pool = LimitedMultiprocessingPool(processes=config.pool_size) + signal.signal(signal.SIGTERM, self.stop) + + def handlePoolCacheExcess(self): + while self.pool.getPoolCacheSize() >= self.pool_cache_limit: + pass + + def run(self, context): + print("consumer is running") + + while not self.should_stop: + for message in self.consumer: + if self.should_stop: + break + + self.callback(context, message.value.decode()) + + def stop(self): + self.should_stop = True diff --git a/modules/message_broker/producer.py b/modules/message_broker/producer.py new file mode 100644 index 0000000..90c87e6 --- /dev/null +++ b/modules/message_broker/producer.py @@ -0,0 +1,14 @@ +import kafka + +class Config: + def __init__(self, server): + self.server = server + +class Producer: + def __init__(self, config): + self.producer = kafka.KafkaProducer( + bootstrap_servers=[config.server] + ) + + def send(self, topic: str, message: str): + self.producer.send(topic, message.encode()) diff --git a/modules/pipelines/pipeline.py b/modules/pipelines/pipeline.py index 3d14f22..b6d3696 100644 --- a/modules/pipelines/pipeline.py +++ b/modules/pipelines/pipeline.py @@ -20,4 +20,7 @@ class Pipeline: for pipe in self.pipes: data = pipe.do(data) + if data is None: + return None + return data diff --git a/modules/pipelines/pipes/preprocessing.py b/modules/pipelines/pipes/preprocessing.py index e9aca30..39eb238 100644 --- a/modules/pipelines/pipes/preprocessing.py +++ b/modules/pipelines/pipes/preprocessing.py @@ -19,6 +19,22 @@ class ToLowerCase(PipeBase): def do(self, data): return data.lower() +class Strip(PipeBase): + """ + Удаляет пробелы в начале и конце строки + """ + + def __init__(self): + """ + Конструктор + """ + + super().__init__("strip", DataType.String, DataType.String) + self.description = "removes any leading and trailing whitespaces" + + def do(self, data: str): + return data.strip() + class Split(PipeBase): """ Разбивает строку на токены через разделитель diff --git a/modules/storages/storage.py b/modules/storages/storage.py index 1e0c166..ba4219a 100644 --- a/modules/storages/storage.py +++ b/modules/storages/storage.py @@ -1,8 +1,16 @@ import psycopg2 as pg +class ConnectionData: + def __init__(self): + self.host = None + self.port = None + self.database = None + self.user = None + self.password = None + class Storage: - def __init__(self, host, port, database, user, password): - self.connection = pg.connect(host=host, port=port, dbname=database, user=user, password=password) + def __init__(self, connection_data): + self.connection = pg.connect(host=connection_data.host, port=connection_data.port, dbname=connection_data.database, user=connection_data.user, password=connection_data.password) def do(self, query, should_return=False): transaction = self.connection.cursor() @@ -12,4 +20,3 @@ class Storage: if should_return: return transaction.fetchall() - diff --git a/modules/transports/server.py b/modules/transports/server.py index 1545619..6485afa 100644 --- a/modules/transports/server.py +++ b/modules/transports/server.py @@ -9,10 +9,6 @@ class Server: self.callback = callback self.context = None self.server = None - self.logger = Logger(LogLevel.Error, None) - - def setLogger(self, max_log_level, callback): - self.logger = Logger(max_log_level, callback) async def run(self): self.server = await asyncio.start_server(self.handleClient, self.host, self.port) @@ -42,11 +38,9 @@ class Server: request = request.decode() - # Вызываем коллбэк response = self.callback(self.context, request) response = struct.pack(">I", len(response)) + response.encode() - # Отправляем ответ writer.write(response) def stop(self): diff --git a/modules/utils/ngrams_dictionary/ngrams_dictionary.py b/modules/utils/ngrams_dictionary/ngrams_dictionary.py new file mode 100644 index 0000000..84cc94b --- /dev/null +++ b/modules/utils/ngrams_dictionary/ngrams_dictionary.py @@ -0,0 +1,26 @@ +class NgramInfo: + def __init__(self, id): + self.id = id + self.frequency = 1 + +class NgramsDictionary: + def __init__(self, ngram_size): + self.ngrams = {} + self.ngram_size = ngram_size + self.last_id = 0 + + def update(self, words): + for last_word_index in range(self.ngram_size, len(words) + 1): + ngram = "" + for current_word_index in range(0, self.ngram_size): + ngram += words[last_word_index - self.ngram_size + current_word_index] + " " + + self.addNgram(ngram[:-1]) + + def addNgram(self, ngram): + if ngram in self.ngrams: + self.ngrams[ngram].frequency += 1 + else: + word_info = NgramInfo(self.last_id) + self.last_id += 1 + self.ngrams[ngram] = word_info diff --git a/modules/utils/pattern.py b/modules/utils/pattern.py new file mode 100644 index 0000000..2832b86 --- /dev/null +++ b/modules/utils/pattern.py @@ -0,0 +1,20 @@ +class Constants: + whitespace = r"\s*" + unknown_begin_part = r"<(\d{1,2})>" + month = r"([A-Za-z]{3})" + number = r"(\d+)" + time = r"(\d{2}:\d{2}:\d{2})" + word = r"(\S+)" + anything = r"(.*)" + +class Pattern: + def __int__(self): + self.parts = [] + + def __str__(self): + result = "" + + for part in self.parts: + result += part[0] + Constants.whitespace + + return result diff --git a/readme.md b/readme.md index 2d3de5b..5998272 100644 --- a/readme.md +++ b/readme.md @@ -12,28 +12,3 @@ Так как модели могут потреблять много ресурсов, они не встраиваются напрямую в пайпы, а работают в отдельном процессе, с которым можно установить соединение (например, через сокет). Таким образом, часть конвейера может выполняться там же, где был получен запрос на обработку, а часть -- на удалённой машине или в другом процессе. - -## Использование -### Сервис Control -Для начала необходимо настроить хранилище, в которую будут сохраняться данные. В качестве хранилища используется PostgreSQL. -В директории sql располагается файл storage.sql. Необходимо: -1. Создать базу данных "log_analysis" -2. В созданной базе данных создать таблицу "pipelines" - -После этого можно запускать сервис Control. При запуске необходимо указать параметры. Список этих параметров можно посмотреть в коде, либо запустить сервис без указания аргументов. -Сервис готов к работе после вывода надписи "server is running" - -К сервису можно подключиться с помощью ConsoleClient (директория services/console_client). При запуске также нужно указать параметры. - -#### Команды -##### Получение пайп -Возвращает список доступных пайп. Этот список формируется при запуске сервиса и в дальнейшем не изменяется. - -##### Получение пайплайнов -Возвращает список доступных пайплайнов. Список может быть дополнен пользовательскими пайпланами. - -##### Создание пайплайна -Создаёт пайплайн из указанных пайп. Пайпы должны соответствовать друг другу (то есть выход предыдущего пайпа должен соответстовать входу следующего) - -##### Запуск пайплайна -Запускает пайплайн для обработки указанной строки. Возвращает результат обработки diff --git a/services/console_client/commands.py b/services/console_client/commands.py deleted file mode 100644 index 287a2ab..0000000 --- a/services/console_client/commands.py +++ /dev/null @@ -1,193 +0,0 @@ -import json - -from modules.transports.client import Client - -class Settings: - def __init__(self): - self.host = None - self.port = None - -class CommandException(Exception): - def __init__(self, message): - super().__init__(message) - -class Command: - def __init__(self, name): - """ - Конструктор - :param name: Название команды - """ - self.name = name - self.settings = None - - def do(self, parameters): - """ - Вызов команды - :param parameters: Параметры - :return: Результат работы - """ - raise Exception("calling unimplemented method") - - def doRequest(self, request: dict): - """ - Выполнение запроса - :param request: Тело запроса - :return: Ответ - """ - - # Создаём клиентский сокет и подключаемся к серверу - client = Client() - client.connect(self.settings.host, self.settings.port) - - # Отправляем запрос - client.send(json.dumps(request)) - - # Получаем ответ - response = client.receive() - - # Закрываем соединение - client.disconnect() - - # Парсим ответ - response = json.loads(response) - - # Проверяем статус обработки запроса - status = response["status"] - if status != "ok": - message = response["message"] - raise CommandException(f"status: {status}, message: {message}") - - return response["result"] - -class CommandManager: - def __init__(self, settings): - self.settings = settings - self.commands = [] - self.command_name_to_index = {} - - def addCommand(self, command: Command): - command.settings = self.settings - self.command_name_to_index[command.name] = len(self.commands) - self.commands.append(command) - - def hasCommand(self, name) -> bool: - if str.isnumeric(name): - index = int(name) - if index < 0 or index >= len(self.commands): - return False - - return True - - return name in self.command_name_to_index - - def getCommand(self, name): - if not self.hasCommand(name): - raise Exception("there is no command with such name") - - if str.isnumeric(name): - return self.commands[int(name)] - - return self.commands[self.command_name_to_index[name]] - -class GetPipes(Command): - """ - Получение списка пайп - """ - - def __init__(self): - super().__init__("get_pipes") - - def do(self, parameters): - request = { - "handler": self.name, - "parameters": "" - } - - # Ответ -- это список пайп - response = self.doRequest(request) - - if len(response) == 0: - print("there are no pipes") - return - - for pipe in response[:-1]: - self.printPipe(pipe) - print() - - self.printPipe(response[-1]) - - def printPipe(self, pipe): - print(pipe["name"] + ":") - print("- description: " + pipe["description"]) - print("- input data type: " + pipe["input_data_type"]) - print("- output data type: " + pipe["output_data_type"]) - -class GetPipelines(Command): - """ - Получение списка пайплайнов - """ - - def __init__(self): - super().__init__("get_pipelines") - - def do(self, parameters): - request = { - "handler": self.name, - "parameters": "" - } - - # Ответ -- это список пайплайнов - response = self.doRequest(request) - - if len(response) == 0: - print("there are no pipelines") - return - - for pipeline in response: - print(pipeline["name"] + ": " + str(pipeline["pipes"])) - -class RunPipeline(Command): - """ - Запуск пайплайна - """ - - def __init__(self): - super().__init__("run_pipeline") - - def do(self, parameters): - pipeline = input("pipeline: ") - data = input("data: ") - - request = { - "handler": self.name, - "parameters": { - "name": pipeline, - "data": data - } - } - - response = self.doRequest(request) - print("result: " + str(response)) - -class CreatePipeline(Command): - """ - Создание пайплайна - """ - - def __init__(self): - super().__init__("create_pipeline") - - def do(self, parameters): - name = input("name: ") - pipes = input("pipes (separated by spaces): ") - - request = { - "handler": self.name, - "parameters": { - "name": name, - "pipes": pipes.split() - } - } - - self.doRequest(request) - print("created successfully") diff --git a/services/console_client/console_client.py b/services/console_client/console_client.py deleted file mode 100644 index 196f7ca..0000000 --- a/services/console_client/console_client.py +++ /dev/null @@ -1,70 +0,0 @@ -import argparse - -from services.console_client.commands import * - -def CreateCommandManager(settings): - manager = CommandManager(settings) - manager.addCommand(GetPipes()) - manager.addCommand(GetPipelines()) - manager.addCommand(CreatePipeline()) - manager.addCommand(RunPipeline()) - - return manager - -def PrintHelp(command_manager): - for index, command in enumerate(command_manager.commands): - print(str(index) + ". " + command.name) - -def ParseArguments(): - parser = argparse.ArgumentParser(description="Console client for Log Analysis") - - # Хост сервера - parser.add_argument("--host", type=str, required=True, help="server's host") - - # Порт - parser.add_argument("--port", type=int, required=True, help="server's port") - - arguments = parser.parse_args() - return arguments - -def CreateSettings() -> Settings: - arguments = ParseArguments() - - settings = Settings() - settings.host = arguments.host - settings.port = arguments.port - - return settings - -def main(): - settings = CreateSettings() - command_manager = CreateCommandManager(settings) - - parameters = {} - - PrintHelp(command_manager) - - while True: - command_name = str(input("> ")) - if command_name in ["q", "quit", "exit"]: - break - - if command_name in ["h", "help"]: - PrintHelp(command_manager) - continue - - if not command_manager.hasCommand(command_name): - print("there is no command with such name") - continue - - command = command_manager.getCommand(command_name) - - try: - command.do(parameters) - except Exception as e: - print(f"error: {str(e)}") - - print() - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/services/control/context.py b/services/control/context.py deleted file mode 100644 index a0dc228..0000000 --- a/services/control/context.py +++ /dev/null @@ -1,19 +0,0 @@ -from modules.pipelines.pipe_manager import PipeManager -from modules.pipelines.pipeline_manager import PipelineManager -from modules.handlers.handler_manager import HandlerManager -from modules.storages.storage import Storage - -class Context: - """ - Контекст управляющего сервиса - """ - - def __init__(self): - """ - Конструктор - """ - - self.pipe_manager = None - self.pipeline_manager = None - self.handler_manager = None - self.storage = None diff --git a/services/control/control.py b/services/control/control.py deleted file mode 100644 index c8235e8..0000000 --- a/services/control/control.py +++ /dev/null @@ -1,130 +0,0 @@ -import argparse -import asyncio -import json - -from modules.logging.logger import * -from modules.storages.storage import Storage -from modules.transports.server import Server -from services.control.context import * -from services.control.create_context import * -from services.control.response import * - -def Handle(context, request): - """ - Обработка запроса - :param context: Контекст сервиса - :param request: Запрос - :return: Ответ - """ - - # Парсим запрос - try: - request = json.loads(request) - except Exception as e: - return ErrorResponse("invalid request", "failed to parse request") - - # Проверяем наличие нужных ключей - for key in ["handler", "parameters"]: - if key not in request: - return ErrorResponse("invalid request", f"key '{key}' is required") - - # Проверяем наличие обработчика с указанным именем - handler_name = request["handler"] - if not context.handler_manager.contains(handler_name): - return ErrorResponse("not found", f"there is no handler with name {handler_name}") - - # Достаём нужного обработчика - handler = context.handler_manager.get(handler_name) - - # Запускаем обработку - try: - result = handler.do(context, request["parameters"]) - return SuccessResponse(result) - except Exception as e: - return ErrorResponse("handling failed", str(e)) - - -def ParseArguments(): - parser = argparse.ArgumentParser(description="Console client for Log Analysis") - - # Данные сервера - parser.add_argument("--host", type=str, required=True, help="server's host") - parser.add_argument("--port", type=int, required=True, help="server's port") - - # Данные для подключения к базе данных - parser.add_argument("--psql-host", type=str, required=True, help="postgresql's host") - parser.add_argument("--psql-port", type=str, required=True, help="postgresql's port") - parser.add_argument("--psql-database", type=str, required=True, help="postgresql's database") - parser.add_argument("--psql-user", type=str, required=True, help="postgresql's user") - parser.add_argument("--psql-password", type=str, required=True, help="postgresql's password") - - arguments = parser.parse_args() - return arguments - -class PostgreSqlSettings: - def __init__(self): - self.host = None - self.port = None - self.database = None - self.user = None - self.password = None - -class Settings: - def __init__(self): - self.host = None - self.port = None - self.postgresql_settings = None - -def CreateSettings(): - arguments = ParseArguments() - - postgresql_settings = PostgreSqlSettings() - postgresql_settings.host = arguments.psql_host - postgresql_settings.port = arguments.psql_port - postgresql_settings.database = arguments.psql_database - postgresql_settings.user = arguments.psql_user - postgresql_settings.password = arguments.psql_password - - settings = Settings() - settings.host = arguments.host - settings.port = arguments.port - settings.postgresql_settings = postgresql_settings - - return settings - -def LogCallback(level, message): - print(f"level: {level}, message: {message}") - -def main(): - settings = CreateSettings() - - server = Server(settings.host, settings.port, Handle) - server.setLogger(LogLevel.Warning, LogCallback) - - # Создаём контекст - context = CreateContext() - - # Подключаемся к хранилищу - context.storage = Storage( - settings.postgresql_settings.host, - settings.postgresql_settings.port, - settings.postgresql_settings.database, - settings.postgresql_settings.user, - settings.postgresql_settings.password - ) - - # Заполняем пайплайны - query = "select name, pipes from pipelines;" - pipelines = context.storage.do(query, should_return=True) - - for name, pipes in pipelines: - context.pipeline_manager.create(name, json.loads(pipes)) - - # Указываем контекст в сервере - server.context = context - - # Запускаем сервер - asyncio.run(server.run()) - -if __name__ == "__main__": - main() diff --git a/services/control/create_context.py b/services/control/create_context.py deleted file mode 100644 index d319ba4..0000000 --- a/services/control/create_context.py +++ /dev/null @@ -1,62 +0,0 @@ -from modules.pipelines.pipe_manager import PipeManager -from modules.pipelines.pipeline_manager import PipelineManager -from modules.pipelines.pipes.preprocessing import * - -from modules.handlers.handler_manager import HandlerManager -from modules.handlers.pipelines.create_pipeline import CreatePipeline -from modules.handlers.pipelines.get_pipelines import GetPipelines -from modules.handlers.pipelines.get_pipes import GetPipes -from modules.handlers.pipelines.run_pipeline import RunPipeline - -from services.control.context import Context - -def CreatePipeManager() -> PipeManager: - """ - Создание заполненного менеджера пайп - :return: Менеджер пайп - """ - - manager = PipeManager() - manager.add(RemovePunctuationMarks()) - manager.add(RemoveStopWords()) - manager.add(Stem()) - manager.add(ToLowerCase()) - manager.add(SplitBySpace()) - - return manager - -def CreatePipelineManager(pipe_manager) -> PipelineManager: - """ - Создание заполненного менеджера пайплайнов - :param pipe_manager: Менеджер пайп - :return: Менеджер пайплайнов - """ - - manager = PipelineManager(pipe_manager) - return manager - -def CreateHandlerManager() -> HandlerManager: - """ - Создание менеджера хэндлеров - :return: Менеджер хэндлеров - """ - manager = HandlerManager() - manager.add(CreatePipeline()) - manager.add(GetPipelines()) - manager.add(GetPipes()) - manager.add(RunPipeline()) - - return manager - -def CreateContext() -> Context: - """ - Создание заполненного контекста - :return: Контекст - """ - - context = Context() - context.pipe_manager = CreatePipeManager() - context.pipeline_manager = CreatePipelineManager(context.pipe_manager) - context.handler_manager = CreateHandlerManager() - - return context diff --git a/services/control/response.py b/services/control/response.py deleted file mode 100644 index f479b74..0000000 --- a/services/control/response.py +++ /dev/null @@ -1,26 +0,0 @@ -import json - -def SuccessResponse(result: str) -> str: - """ - Создание успешного ответа - :param result: Тело ответа - :return: Ответ - """ - - return json.dumps({ - "status": "ok", - "result": result - }) - -def ErrorResponse(status: str, message: str) -> str: - """ - Создание ответа с ошибкой - :param status: Код ошибки - :param message: Сообщение об ошибке - :return: - """ - - return json.dumps({ - "status": status, - "message": message - }) diff --git a/services/dispatchers/all_dispatcher.py b/services/dispatchers/all_dispatcher.py new file mode 100644 index 0000000..b95f378 --- /dev/null +++ b/services/dispatchers/all_dispatcher.py @@ -0,0 +1,41 @@ +import re + +from modules.logging.logger import LogLevel +from modules.utils.pattern import Constants, Pattern +from services.dispatchers.dispatcher_base import DispatcherBase + +def ConsumeCallback(context, message): + match = re.search(str(context.pattern), message) + if match is None: + context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'") + return + + groups = match.groups() + topic = groups[5] + message = groups[6] + + context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'") + context.producer.send(topic, message) + +class AllDispatcher(DispatcherBase): + def __init__(self): + # Паттерн: month number hh:mm:ss machine-name process-name: message + pattern = Pattern() + pattern.parts = [ + (Constants.unknown_begin_part, "unknown begin part"), # 0 + (Constants.month, "month"), # 1 + (Constants.number, "unknown number"), # 2 + (Constants.time, "time"), # 3 + (Constants.word, "machine name"), # 4 + (Constants.word + ":", "process name"), # 5 + (Constants.anything, "message") # 6 + ] + + super().__init__(topic="all", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for all logs") + +def main(): + dispatcher = AllDispatcher() + dispatcher.run() + +if __name__ == "__main__": + main() diff --git a/services/dispatchers/ceph_osd_dispatcher.py b/services/dispatchers/ceph_osd_dispatcher.py new file mode 100644 index 0000000..568a95f --- /dev/null +++ b/services/dispatchers/ceph_osd_dispatcher.py @@ -0,0 +1,43 @@ +import re + +from modules.logging.logger import LogLevel +from modules.utils.pattern import Constants, Pattern +from services.dispatchers.dispatcher_base import DispatcherBase + +def ConsumeCallback(context, message): + match = re.search(str(context.pattern), message) + if match is None: + context.logger.log(LogLevel.Error, f"Nothing is matched for log '{message}'") + return + + groups = match.groups() + topic = groups[3] + message = groups[4] + + context.logger.log(LogLevel.Debug, f"Sending '{message}' to topic '{topic}'") + context.producer.send(topic, message) + +class CephOsdDispatcher(DispatcherBase): + def __init__(self): + # Паттерн: yyyy-mm-ddThh:mm:ss+timezone pid number process-name: message + datetime = Constants.number + "-" + Constants.number + "-" + Constants.number + "T" + Constants.time + "\." + Constants.number + "\+" + Constants.number + datetime = datetime.replace("(", "") + datetime = datetime.replace(")", "") + + pattern = Pattern() + pattern.parts = [ + ("(" + datetime + ")", "Date and time with timezone"), # 0 + (Constants.word, "pid"), # 1 + (Constants.number, "unknown number"), # 2 + (Constants.word + ":", "process name"), # 3 + (Constants.anything, "message") # 4 + ] + + super().__init__(topic="ceph-osd", pattern=pattern, consume_callback=ConsumeCallback, description="Dispatcher for ceph-osd logs") + +def main(): + dispatcher = CephOsdDispatcher() + dispatcher.run() + +if __name__ == "__main__": + main() diff --git a/services/dispatchers/dispatcher_base.py b/services/dispatchers/dispatcher_base.py new file mode 100644 index 0000000..b7c8159 --- /dev/null +++ b/services/dispatchers/dispatcher_base.py @@ -0,0 +1,51 @@ +import argparse +from datetime import datetime + +from modules.logging.logger import LogLevel, Logger +from modules.message_broker.consumer import Config as ConsumerConfig, Consumer +from modules.message_broker.producer import Config as ProducerConfig, Producer +from modules.utils.pattern import Pattern + +class Context: + def __init__(self): + self.logger = None + self.pattern = None + self.producer = None + +def LogCallback(level, message): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"[{str(level)}] {now}: {message}") + +def ParseArguments(description): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--server", type=str, required=True, help="Kafka server") + parser.add_argument("--pool-size", type=int, required=True, help="Pool size") + parser.add_argument("--pool-cache-limit", type=int, required=True, help="Pool cache limit") + parser.add_argument("--max-log-level", type=int, required=False, help="Max log level. Error -- 0 (default), Warning -- 1, Debug -- 2") + + arguments = parser.parse_args() + return arguments + +class DispatcherBase: + def __init__(self, topic: str, pattern: Pattern, consume_callback, description: str): + # Парсим аргументы + arguments = ParseArguments(description) + + # Создаём контекст + self.context = Context() + self.context.pattern = pattern + + # Настраиваем логирование + max_log_level = LogLevel.Debug # todo log level + self.context.logger = Logger(max_log_level=max_log_level, callback=LogCallback) + + # Создаём продюсера + producer_config = ProducerConfig(arguments.server) + self.context.producer = Producer(producer_config) + + # Создаём консюмера + consumer_config = ConsumerConfig(topic, arguments.server, arguments.pool_size, arguments.pool_cache_limit) + self.consumer = Consumer(consumer_config, consume_callback) + + def run(self): + self.consumer.run(self.context) diff --git a/services/log_analyzer/__init__.py b/services/log_analyzer/__init__.py new file mode 100644 index 0000000..3b12ef2 --- /dev/null +++ b/services/log_analyzer/__init__.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/services/log_analyzer/__main__.py b/services/log_analyzer/__main__.py new file mode 100644 index 0000000..bee7425 --- /dev/null +++ b/services/log_analyzer/__main__.py @@ -0,0 +1,138 @@ +import argparse +import json +import re + +from modules.message_broker.consumer import Config, Consumer +from modules.storages.storage import ConnectionData, Storage +from modules.utils.pattern import Constants, Pattern +from services.log_analyzer.model import LogAnalyzer + +class Context: + def __init__(self): + self.kafka_server = None + self.topic = None + self.model = None + self.storage = None + self.message_pattern = None + self.enable_logging = False + +def CheckRequiredKeys(config): + keys = ["storage", "kafka_server", "topic", "vectorization_model", "tokenizer", "model"] + + is_ok = True + for key in keys: + if key not in config: + print(f"config doesn't contain key '{key}'") + is_ok = False + + if not is_ok: + raise RuntimeError("Invalid configuration file") + + storage_config = config["storage"] + storage_keys = ["host", "port", "database", "user", "password"] + for key in storage_keys: + if key not in storage_config: + print(f"storage config doesn't contain key '{key}'") + is_ok = False + + if not is_ok: + raise RuntimeError("Invalid configuration file") + +def ParseConfig() -> Context: + parser = argparse.ArgumentParser("Log analyzer") + parser.add_argument("-c", "--config", required=True, type=str, help="Path to configuration file") + parser.add_argument("-l", "--enable-logging", action="store_true", default=False, help="Enable logging") + + arguments = parser.parse_args() + + with open(arguments.config, "r") as file: + config_data = file.read() + + config = json.loads(config_data) + CheckRequiredKeys(config) + + context = Context() + context.kafka_server = config["kafka_server"] + context.topic = config["topic"] + context.model = LogAnalyzer(config["vectorization_model"], config["tokenizer"]) + context.model.fit(config["model"]) + + storage_config = config["storage"] + connection_data = ConnectionData() + connection_data.host = storage_config["host"] + connection_data.port = storage_config["port"] + connection_data.database = storage_config["database"] + connection_data.user = storage_config["user"] + connection_data.password = storage_config["password"] + + context.storage = Storage(connection_data) + + context.enable_logging = arguments.enable_logging + + pattern = Pattern() + pattern.parts = [ + (Constants.unknown_begin_part, "unknown begin part"), # 0 + (Constants.month, "month"), # 1 + (Constants.number, "unknown number"), # 2 + (Constants.time, "time"), # 3 + (Constants.word, "machine name"), # 4 + (Constants.word + ":", "process name"), # 5 + (Constants.anything, "message") # 6 + ] + + context.message_pattern = re.compile(str(pattern)) + + return context + +def ClearMessage(context: Context, message: str) -> tuple: + match = re.search(context.message_pattern, message) + if match is None: + if context.enable_logging: + print(f"Nothing is matched for log '{message}'") + + return "", message + + groups = match.groups() + topic = groups[5] + message = groups[6] + + return topic, message + +def UpdateStatisticsQuery(name: str): + return f"update statistics set value = value + 1 where name = '{name}';" + +def SaveAnomalyLogQuery(log: str): + return f"insert into anomaly_logs (data) values('{log}')" + +def Callback(context: Context, message: str): + if context.enable_logging: + print(f"Processing message '{message}'", message) + + topic, message = ClearMessage(context, message) + + result = context.model.predict(program=topic, message=message)[0] + result_as_str = "anomaly" if result == -1 else "not anomaly" + + query = UpdateStatisticsQuery("total") + UpdateStatisticsQuery(result_as_str) + context.storage.do(query) + + if result == -1: + query = SaveAnomalyLogQuery(message) + context.storage.do(query) + + if context.enable_logging: + print("log is " + result_as_str) + +def main(): + # Создаём контекст + context = ParseConfig() + + # Создаём потребителя + config = Config(context.topic, context.kafka_server, 16, 16) + consumer = Consumer(config, Callback) + + # Запускаем обработку + consumer.run(context) + +if __name__ == "__main__": + main() diff --git a/services/log_analyzer/model.py b/services/log_analyzer/model.py new file mode 100644 index 0000000..5c2d27b --- /dev/null +++ b/services/log_analyzer/model.py @@ -0,0 +1,86 @@ +from transformers import AutoTokenizer, AutoModel, AutoConfig +import torch +import torch.nn.functional as F +import pickle +import numpy +import pandas as pd + +class LogAnalyzer: + '''Сlass of an expert system module that analyzes diagnostic information + about the operation of the input-output subsystem. + ''' + + def __init__(self, path_model_embedding: str, path_tokenizer: str): + '''Constructor of model for analyse. + :param path_model_embedding: String of path to directory with model + for build embeddings of message + :param path_tokenizer: String of path to directory with tokenizer + for tokenize message + ''' + self.tokenizer = AutoTokenizer.from_pretrained(path_tokenizer) + self.model_embedding = AutoModel.from_pretrained(path_model_embedding) + self.model = None + + def predict(self, program: str, message: str) -> numpy.ndarray: + '''Function predict whether an log is anomaly. + :param message: string of message + ''' + + def mean_pooling(model_output, attention_mask) -> torch.Tensor: + '''Function applies mean pooling on the embedding model to get + a single vector representation. + :param model_output: Tuple containing different model outputs. + The first element contains the token embeddings. + :param attention_mask: Tensor that identifies the tokens + to which attention should be paid (value of 1) + versus those which should not (value of 0). + :return: Tensor representing the pooled output of the input tokens. + ''' + token_embeddings = model_output[0] + input_mask_expanded = attention_mask.unsqueeze(-1).expand( + token_embeddings.size()).float() + return torch.sum( + token_embeddings * input_mask_expanded, 1) / torch.clamp( + input_mask_expanded.sum(1), min=1e-9) + + def embed(message: str, tokenizer, model) -> torch.Tensor: + '''Function generates embeddings for a given message. + :param message: String of message for which embeddings + are to be generated. + :param tokenizer: The tokenizer to be used for tokenizing + the message. + :param model: The model to be used for generating embeddings. + :return: Tensor of normalized embeddings of the input message. + ''' + encoded_input = tokenizer( + message, padding=True, truncation=True, return_tensors='pt') + + with torch.no_grad(): + model_output = model(**encoded_input) + + message_embeddings_mean = mean_pooling( + model_output, encoded_input['attention_mask']) + message_embeddings = F.normalize( + message_embeddings_mean, p=2, dim=1) + return message_embeddings + + if self.model is None: + raise ValueError("Model is not fitted. Make '.fit' method first") + return None + + message_embeddings = embed( + message, self.tokenizer, self.model_embedding) + + try: + prediction = self.model[program].predict( + message_embeddings.numpy()) + except KeyError: + prediction = self.model['unknown'].predict( + message_embeddings.numpy()) + return prediction + + def fit(self, path_model): + '''Function predict whether an log is anomaly. + :param path_model: String of path to saved model + ''' + self.model = pickle.load(open(path_model, 'rb')) diff --git a/services/rocksdb_analyzer/pipes.py b/services/rocksdb_analyzer/pipes.py new file mode 100644 index 0000000..aa13dec --- /dev/null +++ b/services/rocksdb_analyzer/pipes.py @@ -0,0 +1,33 @@ +import re + +from modules.pipelines.pipes.preprocessing import DataType, PipeBase + +class RemoveByRegex(PipeBase): + def __init__(self, name, pattern): + super().__init__(name, DataType.String, DataType.String) + + self.pattern = re.compile(pattern) + + def do(self, data): + return self.pattern.sub("", data) + +class RemoveOriginalLogTime(RemoveByRegex): + def __init__(self): + super().__init__("remove_original_log_time", "\(Original Log Time .*\)") + +class RemoveSourceCode(RemoveByRegex): + def __init__(self): + super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]") + +class StopRunningIfContains(PipeBase): + def __init__(self, items: list[str]): + super().__init__("stop_running_if_contains", DataType.String, DataType.String) + + self.items = items + + def do(self, data: str): + for item in self.items: + if data.find(item) != -1: + return None + + return data diff --git a/services/rocksdb_analyzer/rocksdb_analyzer.py b/services/rocksdb_analyzer/rocksdb_analyzer.py new file mode 100644 index 0000000..c32fac5 --- /dev/null +++ b/services/rocksdb_analyzer/rocksdb_analyzer.py @@ -0,0 +1,22 @@ +from modules.pipelines.pipeline import Pipeline +from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace +from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains + +def main(): + pipeline = Pipeline("prepare") + pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"])) + pipeline.add(RemoveOriginalLogTime()) + pipeline.add(RemoveSourceCode()) + pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]")) + pipeline.add(Strip()) + + with open("rocksdb_logs.txt", "r") as input: + with open("filtered_rocksdb_logs.txt", "w") as output: + for line in input.readlines(): + result = pipeline.run(line) + + if result is not None: + output.write(result + "\n") + +if __name__ == "__main__": + main() diff --git a/simple_consumer.py b/simple_consumer.py new file mode 100644 index 0000000..2436e6a --- /dev/null +++ b/simple_consumer.py @@ -0,0 +1,13 @@ +import argparse +from kafka import KafkaConsumer + +parser = argparse.ArgumentParser() +parser.add_argument("--topic", type=str, required=True) + +arguments = parser.parse_args() + +consumer = KafkaConsumer(bootstrap_servers=["localhost:29092"]) +consumer.subscribe(arguments.topic) + +for message in consumer: + print(message.value) diff --git a/simple_producer.py b/simple_producer.py new file mode 100644 index 0000000..4e3d725 --- /dev/null +++ b/simple_producer.py @@ -0,0 +1,12 @@ +import argparse +from kafka import KafkaProducer + +parser = argparse.ArgumentParser() +parser.add_argument("--topic", type=str, required=True) +parser.add_argument("--message", type=str, required=True) + +arguments = parser.parse_args() + +producer = KafkaProducer(bootstrap_servers=["localhost:29092"]) +future = producer.send(arguments.topic, arguments.message.encode()) +future.get(timeout=3) diff --git a/sql/storage.sql b/sql/storage.sql deleted file mode 100644 index ce93349..0000000 --- a/sql/storage.sql +++ /dev/null @@ -1,7 +0,0 @@ -create database log_analysis; - -create table pipelines( - id serial, - name text, - pipes text -); diff --git a/sql/tables.sql b/sql/tables.sql new file mode 100644 index 0000000..a357a23 --- /dev/null +++ b/sql/tables.sql @@ -0,0 +1,15 @@ +create database log_analysis; + +create table statistics( + name varchar(64), + value int, + primary key (name) +); + +insert into statistics (name, value) values ('total', 0), ('anomaly', 0), ('not anomaly', 0), ('undefined', 0); + +create table anomaly_logs( + id serial, + data text, + primary key (id) +);