Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+45
View File
@@ -0,0 +1,45 @@
import signal
import kafka
import kafka.errors
import multiprocessing.pool
class Config:
def __init__(self, topic, server, pool_size, pool_cache_limit):
self.topic = topic
self.server = server
self.pool_size = pool_size
self.pool_cache_limit = pool_cache_limit
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
def getPoolCacheSize(self):
return len(self._cache)
class Consumer:
def __init__(self, config: Config, callback):
self.callback = callback
self.consumer = kafka.KafkaConsumer(
config.topic,
bootstrap_servers=[config.server],
)
self.should_stop = False
self.pool_cache_limit = config.pool_cache_limit
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
signal.signal(signal.SIGTERM, self.stop)
def handlePoolCacheExcess(self):
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
pass
def run(self, context):
print("consumer is running")
while not self.should_stop:
for message in self.consumer:
if self.should_stop:
break
self.callback(context, message.value.decode())
def stop(self):
self.should_stop = True
+14
View File
@@ -0,0 +1,14 @@
import kafka
class Config:
def __init__(self, server):
self.server = server
class Producer:
def __init__(self, config):
self.producer = kafka.KafkaProducer(
bootstrap_servers=[config.server]
)
def send(self, topic: str, message: str):
self.producer.send(topic, message.encode())