Files
log_analysis/modules/message_broker/consumer.py
T

46 lines
1.3 KiB
Python

import signal
import kafka
import kafka.errors
import multiprocessing.pool
class Config:
def __init__(self, topic, server, pool_size, pool_cache_limit):
self.topic = topic
self.server = server
self.pool_size = pool_size
self.pool_cache_limit = pool_cache_limit
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
def getPoolCacheSize(self):
return len(self._cache)
class Consumer:
def __init__(self, config: Config, callback):
self.callback = callback
self.consumer = kafka.KafkaConsumer(
config.topic,
bootstrap_servers=[config.server],
)
self.should_stop = False
self.pool_cache_limit = config.pool_cache_limit
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
signal.signal(signal.SIGTERM, self.stop)
def handlePoolCacheExcess(self):
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
pass
def run(self, context):
print("consumer is running")
while not self.should_stop:
for message in self.consumer:
if self.should_stop:
break
self.callback(context, message.value.decode())
def stop(self):
self.should_stop = True