Большое обновление всего проекта
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
import signal
|
||||
import kafka
|
||||
import kafka.errors
|
||||
import multiprocessing.pool
|
||||
|
||||
class Config:
|
||||
def __init__(self, topic, server, pool_size, pool_cache_limit):
|
||||
self.topic = topic
|
||||
self.server = server
|
||||
self.pool_size = pool_size
|
||||
self.pool_cache_limit = pool_cache_limit
|
||||
|
||||
class LimitedMultiprocessingPool(multiprocessing.pool.Pool):
|
||||
def getPoolCacheSize(self):
|
||||
return len(self._cache)
|
||||
|
||||
class Consumer:
|
||||
def __init__(self, config: Config, callback):
|
||||
self.callback = callback
|
||||
self.consumer = kafka.KafkaConsumer(
|
||||
config.topic,
|
||||
bootstrap_servers=[config.server],
|
||||
)
|
||||
|
||||
self.should_stop = False
|
||||
self.pool_cache_limit = config.pool_cache_limit
|
||||
self.pool = LimitedMultiprocessingPool(processes=config.pool_size)
|
||||
signal.signal(signal.SIGTERM, self.stop)
|
||||
|
||||
def handlePoolCacheExcess(self):
|
||||
while self.pool.getPoolCacheSize() >= self.pool_cache_limit:
|
||||
pass
|
||||
|
||||
def run(self, context):
|
||||
print("consumer is running")
|
||||
|
||||
while not self.should_stop:
|
||||
for message in self.consumer:
|
||||
if self.should_stop:
|
||||
break
|
||||
|
||||
self.callback(context, message.value.decode())
|
||||
|
||||
def stop(self):
|
||||
self.should_stop = True
|
||||
@@ -0,0 +1,14 @@
|
||||
import kafka
|
||||
|
||||
class Config:
|
||||
def __init__(self, server):
|
||||
self.server = server
|
||||
|
||||
class Producer:
|
||||
def __init__(self, config):
|
||||
self.producer = kafka.KafkaProducer(
|
||||
bootstrap_servers=[config.server]
|
||||
)
|
||||
|
||||
def send(self, topic: str, message: str):
|
||||
self.producer.send(topic, message.encode())
|
||||
Reference in New Issue
Block a user