46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
import signal
|
|
import kafka
|
|
import kafka.errors
|
|
import multiprocessing.pool
|
|
|
|
class Config:
|
|
def __init__(self, topic, server, pool_size, pool_cache_limit):
|
|
self.topic = topic
|
|
self.server = server
|
|
self.pool_size = pool_size
|
|
self.pool_cache_limit = pool_cache_limit
|
|
|
|
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
|
|
def getPoolCacheSize(self):
|
|
return len(self._cache)
|
|
|
|
class Consumer:
|
|
def __init__(self, config: Config, callback):
|
|
self.callback = callback
|
|
self.consumer = kafka.KafkaConsumer(
|
|
config.topic,
|
|
bootstrap_servers=[config.server],
|
|
)
|
|
|
|
self.should_stop = False
|
|
self.pool_cache_limit = config.pool_cache_limit
|
|
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
|
|
signal.signal(signal.SIGTERM, self.stop)
|
|
|
|
def handlePoolCacheExcess(self):
|
|
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
|
|
pass
|
|
|
|
def run(self, context):
|
|
print("consumer is running")
|
|
|
|
while not self.should_stop:
|
|
for message in self.consumer:
|
|
if self.should_stop:
|
|
break
|
|
|
|
self.callback(context, message.value.decode())
|
|
|
|
def stop(self):
|
|
self.should_stop = True
|