Demo tool for finding anomalies in RRD files

This commit is contained in:
Luca Deri
2021-12-03 17:36:49 +01:00
parent 3dd9b5176e
commit 1b077ba65f
4 changed files with 173 additions and 1 deletions
+7
View File
@@ -0,0 +1,7 @@
INC=-I ../src/include
LIBDPI=../src/lib/libndpi.a
LIB=$(LIBDPI) -lrrd -lm -lmaxminddb -lgcrypt
rrd_anomaly: rrd_anomaly.c Makefile $(LIBDPI)
$(CC) -g $(INC) rrd_anomaly.c -o rrd_anomaly $(LIB)
+6
View File
@@ -0,0 +1,6 @@
This directory contains a tool that allows to identify anomalies in RRD files
Prerequisite
- rrdtool (https://oss.oetiker.ch/rrdtool/)
- apt-get install rrdtool
+159
View File
@@ -0,0 +1,159 @@
/*
* ndpiReader.c
*
* Copyright (C) 2011-21 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <getopt.h>
#include "rrd.h"
#include "ndpi_api.h"
#define DEFAULT_ALPHA 0.5
#define DEFAULT_START "now-1d"
#define DEFAULT_END "now"
/* *************************************************** */
static void help() {
printf("Usage: rrd_anomaly [-a <alpha>][-s <start>][-e <end>] -f <filename>\n"
"-a | Set alpha. Valid range >0 .. <1. Default %.2f\n"
"-s <start> | RRD start time. Default %s\n"
"-e <end> | RRD end time. Default %s\n"
"-f <rrd path> | Path of the RRD filename to analyze\n",
DEFAULT_ALPHA, DEFAULT_START, DEFAULT_END);
printf("\n\nExample: rrd_anomaly -s now-1d -e now -f hum.rrd\n");
exit(0);
}
/* *************************************************** */
int main(int argc, char *argv[]) {
rrd_time_value_t start_tv, end_tv;
unsigned long step = 0, ds_cnt = 0;
rrd_value_t *data, *p;
char **names, *filename = NULL, *start_s, *end_s, *cf;
u_int i, j, t, first = 1;
time_t start, end;
struct ndpi_ses_struct ses;
float alpha;
char c;
/* Defaults */
alpha = DEFAULT_ALPHA;
start_s = DEFAULT_START;
end_s = DEFAULT_END;
cf = "AVERAGE";
while((c = getopt(argc, argv, "s:e:a:f:")) != '?') {
if(c == -1) break;
switch(c) {
case 's':
start_s = optarg;
break;
case 'e':
end_s = optarg;
break;
case 'a':
{
float f = atof(optarg);
if((f > 0) && (f < 1))
alpha = f;
else
printf("Discarding -a: valid range is >0 .. <1\n");
}
break;
case 'f':
filename = optarg;
break;
default:
help();
break;
}
}
if(filename == NULL)
help();
ndpi_ses_init(&ses, alpha, 0.05);
if((rrd_parsetime(start_s, &start_tv) != NULL)) {
printf("Unable to parse start time %s\n", start_s);
return(-1);
}
if((rrd_parsetime(end_s, &end_tv) != NULL)) {
printf("Unable to parse end time %s\n", end_s);
return(-1);
}
rrd_proc_start_end(&start_tv, &end_tv, &start, &end);
if(rrd_fetch_r(filename, cf, &start, &end, &step, &ds_cnt, &names, &data) != 0) {
printf("Unable to extract data from rrd %s\n", filename);
return(-2);
}
p = data;
for(t=start+1, i=0; t<end; t+=step, i++) {
for(j=0; j<ds_cnt; j++) {
rrd_value_t value = *p++;
if(!isnan(value)) {
double prediction, confidence_band;
double lower, upper;
char buf[32];
int rc;
u_int is_anomaly;
value *= 100;
rc = ndpi_ses_add_value(&ses, value, &prediction, &confidence_band);
lower = prediction - confidence_band, upper = prediction + confidence_band;
is_anomaly = ((rc == 0) || ((value >= lower) && (value <= upper))) ? 0 : 1;
if(is_anomaly) {
struct tm *t_info = localtime((const time_t*)&t);
strftime(buf, sizeof(buf), "%d/%b/%Y %H:%M:%S", t_info);
if(first) {
first = 0;
printf("%s %s\t%s %s %s\t %s [%s]\n",
"When", "Value", "Prediction", "Lower", "Upper", "Out", "Band");
}
printf("%s %12.3f\t%.3f\t%12.3f\t%12.3f\t %s [%.3f]\n",
buf, value/100., prediction/100., lower/100., upper/100., is_anomaly? "ANOMALY" : "OK",
confidence_band/100.);
}
}
}
}
rrd_freemem(data);
return(0);
}
+1 -1
View File
@@ -1226,7 +1226,7 @@ int ndpi_ses_add_value(struct ndpi_ses_struct *ses, const u_int64_t _value, doub
#ifdef SES_DEBUG
printf("[num_values: %u][[error: %.3f][forecast: %.3f][sqe: %.3f][sq: %.3f][confidence_band: %.3f]\n",
ses->num_values, error, *forecast, ses->sum_square_error, sq, *confidence_band);
ses->num_values, error, *forecast, ses->sum_square_error, sq_error, *confidence_band);
#endif
return(rc);