Improved bin clustering

This commit is contained in:
Luca Deri
2020-07-22 23:56:50 +02:00
parent 0956badd11
commit 439558f6a3
3 changed files with 93 additions and 27 deletions
+1 -1
View File
@@ -2509,7 +2509,7 @@ static void printFlowsStats() {
all_flows[i].flow->detected_protocol, buf, sizeof(buf)),
all_flows[i].flow->src_name,
ntohs(all_flows[i].flow->src_port),
all_flows[i].flow->src_name,
all_flows[i].flow->dst_name,
ntohs(all_flows[i].flow->dst_port));
print_bin(out, NULL, &bins[i]);
+90 -24
View File
@@ -78,7 +78,7 @@ void ndpi_free_data_analysis(struct ndpi_analyze_struct *d) {
void ndpi_reset_data_analysis(struct ndpi_analyze_struct *d) {
memset(d, 0, sizeof(struct ndpi_analyze_struct));
memset(d->values, 0, sizeof(u_int32_t)*d->num_values_array_len);
d->num_data_entries = 0;
d->num_data_entries = 0;
}
/* ********************************************************************************* */
@@ -123,11 +123,11 @@ float ndpi_data_average(struct ndpi_analyze_struct *s) {
u_int32_t ndpi_data_last(struct ndpi_analyze_struct *s) {
if((s->num_data_entries == 0) || (s->sum_total == 0))
return(0);
if(s->next_value_insert_index == 0)
return(s->values[s->num_values_array_len-1]);
else
return(s->values[s->next_value_insert_index-1]);
return(s->values[s->next_value_insert_index-1]);
}
/* Return min/max on all values */
@@ -164,7 +164,7 @@ float ndpi_data_window_average(struct ndpi_analyze_struct *s) {
if(n == 0)
return(0);
for(i=0; i<n; i++)
sum += s->values[i];
@@ -183,7 +183,7 @@ float ndpi_data_window_variance(struct ndpi_analyze_struct *s) {
if(n == 0)
return(0);
for(i=0; i<n; i++)
sum += pow(s->values[i]-avg, 2);
@@ -393,7 +393,7 @@ void ndpi_set_bin(struct ndpi_bin *b, u_int8_t slot_id, u_int32_t val) {
void ndpi_inc_bin(struct ndpi_bin *b, u_int8_t slot_id, u_int32_t val) {
b->is_empty = 0;
if(slot_id >= b->num_bins) slot_id = 0;
switch(b->family) {
@@ -456,7 +456,7 @@ void ndpi_normalize_bin(struct ndpi_bin *b) {
u_int32_t tot = 0;
if(b->is_empty) return;
switch(b->family) {
case ndpi_bin_family8:
for(i=0; i<b->num_bins; i++) tot += b->u.bins8[i];
@@ -567,10 +567,10 @@ float ndpi_bin_similarity(struct ndpi_bin *b1, struct ndpi_bin *b2, u_int8_t nor
for(i=0; i<b1->num_bins; i++) {
u_int32_t a = ndpi_get_bin_value(b1, i);
u_int32_t b = ndpi_get_bin_value(b2, i);
sumxx += a*a, sumyy += b*b, sumxy += a*b;
}
if((sumxx == 0) || (sumyy == 0))
return(0);
else
@@ -586,7 +586,7 @@ float ndpi_bin_similarity(struct ndpi_bin *b1, struct ndpi_bin *b2, u_int8_t nor
sum += pow(a-b, 2);
}
/* The lower the more similar */
return(sqrt(sum));
}
@@ -595,6 +595,8 @@ float ndpi_bin_similarity(struct ndpi_bin *b1, struct ndpi_bin *b2, u_int8_t nor
/* ********************************************************************************* */
#define MAX_NUM_CLUSTERS 128
/*
Clusters bins into 'num_clusters'
- (in) bins: a vection 'num_bins' long of bins to cluster
@@ -610,18 +612,25 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
u_int16_t i, j, max_iterations = 25, num_iterations, num_moves;
u_int8_t verbose = 0, alloc_centroids = 0;
char out_buf[256];
if(num_clusters > num_bins) num_clusters = num_bins;
float *bin_score;
u_int16_t num_cluster_elems[MAX_NUM_CLUSTERS] = { 0 };
if(num_clusters > num_bins) num_clusters = num_bins;
if(num_clusters > MAX_NUM_CLUSTERS) num_clusters = MAX_NUM_CLUSTERS;
if(verbose)
printf("Distributing %u bins over %u clusters\n", num_bins, num_clusters);
if((bin_score = (float*)ndpi_calloc(num_bins, sizeof(float))) == NULL)
return(-2);
if(centroids == NULL) {
alloc_centroids = 1;
if((centroids = (struct ndpi_bin*)ndpi_malloc(sizeof(struct ndpi_bin)*num_clusters)) == NULL)
if((centroids = (struct ndpi_bin*)ndpi_malloc(sizeof(struct ndpi_bin)*num_clusters)) == NULL) {
ndpi_free(bin_score);
return(-2);
else {
} else {
for(i=0; i<num_clusters; i++)
ndpi_init_bin(&centroids[i], ndpi_bin_family32 /* Use 32 bit to avoid overlaps */, bins[0].num_bins);
}
@@ -638,6 +647,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
printf("Initializing cluster %u: %s\n", i,
ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf)));
num_cluster_elems[i]++;
}
/* Assign the remaining bins to the nearest cluster */
@@ -668,6 +678,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf)), best_similarity);
cluster_ids[i] = cluster_id;
num_cluster_elems[cluster_id]++;
}
num_iterations = 0;
@@ -675,9 +686,15 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
/* Now let's try to find a better arrangement */
while(num_iterations++ < max_iterations) {
/* Find the center of each cluster */
memset(bin_score, 0, num_bins*sizeof(float));
if(verbose) printf("Iteration %u\n", num_iterations);
if(verbose) {
printf("\nIteration %u\n", num_iterations);
for(j=0; j<num_clusters; j++)
printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
}
for(i=0; i<num_clusters; i++)
ndpi_reset_bin(&centroids[i]);
@@ -704,34 +721,42 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
u_int8_t cluster_id = 0;
#ifdef COSINE_SIMILARITY
best_similarity = -1;
best_similarity = -1;
#else
best_similarity = 99999999999;
best_similarity = 99999999999;
#endif
for(j=0; j<num_clusters; j++) {
float similarity;
if(centroids[j].is_empty) continue;
similarity = ndpi_bin_similarity(&bins[i], &centroids[j], 0);
if(verbose)
printf("Bin %u / centroid %u [similarity: %f]\n", i, j, similarity);
#ifdef COSINE_SIMILARITY
if(similarity > best_similarity)
#else
if(similarity < best_similarity)
#endif
if(similarity > best_similarity) {
cluster_id = j, best_similarity = similarity;
}
#else
if(similarity < best_similarity) {
cluster_id = j, best_similarity = similarity;
}
#endif
}
bin_score[i] = best_similarity;
if(/* (best_similarity > 0) && */ (cluster_ids[i] != cluster_id)) {
if(verbose)
printf("Moved bin %u from cluster %u -> %u [similarity: %f]\n",
i, cluster_ids[i], cluster_id, best_similarity);
num_cluster_elems[cluster_ids[i]]--;
num_cluster_elems[cluster_id]++;
cluster_ids[i] = cluster_id;
num_moves++;
}
@@ -739,7 +764,46 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
if(num_moves == 0)
break;
}
if(verbose) {
for(j=0; j<num_clusters; j++)
printf("Cluster %u: %u bins\n", j, num_cluster_elems[j]);
}
for(j=0; j<num_clusters; j++) {
if(num_cluster_elems[j] == 0) {
u_int16_t candidate;
float score;
if(verbose)
printf("\nCluster %u is empty: need to rebalance\n", j);
#ifdef COSINE_SIMILARITY
score = 99999999999;
for(i=0; i<num_bins; i++) {
if((cluster_ids[i] != j) && (bin_score[i] < score) && (num_cluster_elems[cluster_ids[i]] > 1))
score = bin_score[i], candidate = i;
}
#else
score = 0;
for(i=0; i<num_bins; i++) {
if((cluster_ids[i] != j) && (bin_score[i] > score) && (num_cluster_elems[cluster_ids[i]] > 1))
score = bin_score[i], candidate = i;
}
#endif
if(verbose)
printf("Rebalance: moving bin %u from cluster %u -> %u [similarity: %f]\n",
candidate, cluster_ids[candidate], j, score);
num_cluster_elems[cluster_ids[candidate]]--;
num_cluster_elems[j]++;
cluster_ids[candidate] = j;
}
}
} /* while(...) */
if(alloc_centroids) {
for(i=0; i<num_clusters; i++)
@@ -748,6 +812,8 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
ndpi_free(centroids);
}
ndpi_free(bin_score);
return(0);
}
+2 -2
View File
@@ -1035,8 +1035,8 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
ndpi_build_default_ports(ports_a, 5900, 5901, 5800, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE90, 0 /* can_have_a_subprotocol */,
no_master, no_master, "Free90", NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS,
ndpi_build_default_ports(ports_a, 5900, 5901, 5800, 0, 0) /* TCP */,
no_master, no_master, "FREE_90", NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_ZOOM, 0 /* can_have_a_subprotocol */,
no_master, no_master, "Zoom", NDPI_PROTOCOL_CATEGORY_VIDEO,