Большое обновление всего проекта

This commit is contained in:
2024-01-15 03:45:18 +03:00
parent 827674ba4c
commit 026cc9b6bb
36 changed files with 760 additions and 617 deletions
+33
View File
@@ -0,0 +1,33 @@
import re
from modules.pipelines.pipes.preprocessing import DataType, PipeBase
class RemoveByRegex(PipeBase):
def __init__(self, name, pattern):
super().__init__(name, DataType.String, DataType.String)
self.pattern = re.compile(pattern)
def do(self, data):
return self.pattern.sub("", data)
class RemoveOriginalLogTime(RemoveByRegex):
def __init__(self):
super().__init__("remove_original_log_time", "\(Original Log Time .*\)")
class RemoveSourceCode(RemoveByRegex):
def __init__(self):
super().__init__("remove_source_code", "\[\w*\/?\w*\.\w*:\d+\]")
class StopRunningIfContains(PipeBase):
def __init__(self, items: list[str]):
super().__init__("stop_running_if_contains", DataType.String, DataType.String)
self.items = items
def do(self, data: str):
for item in self.items:
if data.find(item) != -1:
return None
return data
@@ -0,0 +1,22 @@
from modules.pipelines.pipeline import Pipeline
from modules.pipelines.pipes.preprocessing import Strip, SplitBySpace
from services.rocksdb_analyzer.pipes import RemoveByRegex, RemoveOriginalLogTime, RemoveSourceCode, StopRunningIfContains
def main():
pipeline = Pipeline("prepare")
pipeline.add(StopRunningIfContains(["EVENT_LOG_v1", "DB Stats", "DUMPING STATS"]))
pipeline.add(RemoveOriginalLogTime())
pipeline.add(RemoveSourceCode())
pipeline.add(RemoveByRegex("remove_insignificant_tokens", "\[[A-Za-z]+\]|\[[A-Za-z]-\d+\]|\[JOB \d+\]"))
pipeline.add(Strip())
with open("rocksdb_logs.txt", "r") as input:
with open("filtered_rocksdb_logs.txt", "w") as output:
for line in input.readlines():
result = pipeline.run(line)
if result is not None:
output.write(result + "\n")
if __name__ == "__main__":
main()