Removed this party LRU and replaced with home grown

This commit is contained in:
Luca Deri
2019-01-17 00:40:28 +01:00
parent c856e0a563
commit efef99cbad
10 changed files with 79 additions and 377 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
library_includedir=$(includedir)/libndpi-@VERSION@/libndpi
plibrary_includedir=$(includedir)/libndpi-@VERSION@/libndpi
library_include_HEADERS = ndpi_api.h \
ndpi_define.h \
+6
View File
@@ -756,6 +756,12 @@ extern "C" {
u_int ndpi_get_ndpi_num_custom_protocols(struct ndpi_detection_module_struct *ndpi_mod);
u_int ndpi_get_ndpi_detection_module_size();
void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l);
/* LRU cache */
struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries);
void ndpi_lru_free_cache(struct ndpi_lru_cache *c);
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found);
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key);
/**
* Add a string to match to an automata
+5 -1
View File
@@ -367,6 +367,10 @@ typedef enum {
HTTP_METHOD_CONNECT
} ndpi_http_method;
struct ndpi_lru_cache {
u_int32_t num_entries, *entries;
};
struct ndpi_id_struct {
/**
detected_protocol_bitmask:
@@ -997,7 +1001,7 @@ struct ndpi_detection_module_struct {
int bt_ann_len;
/* NDPI_PROTOCOL_OOKLA */
void *ookla_cache;
struct ndpi_lru_cache *ookla_cache;
/* NDPI_PROTOCOL_TINC */
struct cache *tinc_cache;
+46 -2
View File
@@ -30,7 +30,6 @@
#include <sys/types.h>
#include "ahocorasick.h"
#include "libcache.h"
#include "lruc.h"
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN
@@ -2433,7 +2432,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
cache_free((cache_t)(ndpi_struct->tinc_cache));
if(ndpi_struct->ookla_cache)
lruc_free((lruc*)ndpi_struct->ookla_cache);
ndpi_lru_free_cache(ndpi_struct->ookla_cache);
if(ndpi_struct->protocols_ptree)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data);
@@ -3995,6 +3994,9 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
if(flow == NULL) return(ret);
if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest)))
printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest));
/* TODO: add the remaining stage_XXXX protocols */
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
u_int16_t guessed_protocol_id, guessed_host_protocol_id;
@@ -6171,6 +6173,48 @@ void ndpi_set_log_level(struct ndpi_detection_module_struct *ndpi_mod, u_int l)
/* ******************************************************************** */
/* LRU cache */
struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) {
struct ndpi_lru_cache *c = (struct ndpi_lru_cache*)malloc(sizeof(struct ndpi_lru_cache));
if(!c) return(NULL);
c->entries = (u_int32_t*)calloc(num_entries, sizeof(u_int32_t));
if(!c->entries) {
free(c);
return(NULL);
} else
c->num_entries = num_entries;
return(c);
}
void ndpi_lru_free_cache(struct ndpi_lru_cache *c) {
free(c->entries);
free(c);
}
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found) {
u_int32_t slot = key % c->num_entries;
if(c->entries[slot] == key) {
if(clean_key_when_found) c->entries[slot] = 0;
return(1);
} else
return(0);
}
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) {
u_int32_t slot = key % c->num_entries;
c->entries[slot] = key;
}
/* ******************************************************************** */
/*
NOTE:
- Leave fields empty/zero when information is missing (e.g. with ICMP ports are zero)
+7 -12
View File
@@ -26,7 +26,6 @@
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP
#include "ndpi_api.h"
#include "lruc.h"
static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
@@ -642,19 +641,15 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
if(ndpi_struct->ookla_cache == NULL)
ndpi_struct->ookla_cache = lruc_new(4*1024, 1024);
ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024);
if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) {
u_int8_t *dummy = (u_int8_t*)ndpi_malloc(sizeof(u_int8_t));
if(dummy) {
if(packet->tcp->source == htons(8080))
lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->saddr, 4, dummy, 1);
else
lruc_set((lruc*)ndpi_struct->ookla_cache, (void*)&packet->iph->daddr, 4, dummy, 1);
}
if(packet->tcp->source == htons(8080))
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr);
else
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr);
}
return;
}
+6 -11
View File
@@ -22,36 +22,32 @@
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_OOKLA
#include "ndpi_api.h"
#include "lruc.h"
void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
#if 0
struct ndpi_packet_struct* packet = &flow->packet;
u_int32_t addr = 0;
void *value;
NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
if(packet->tcp->source == htons(8080))
addr = packet->iph->saddr;
else if(packet->tcp->dest == htons(8080))
addr = packet->iph->daddr;
else
goto ookla_exclude;
if(ndpi_struct->ookla_cache != NULL) {
if(lruc_get(ndpi_struct->ookla_cache, &addr, sizeof(addr), &value) == LRUC_NO_ERROR) {
/* Don't remove it as it can be used for other connections */
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, 0 /* Don't remove it as it can be used for other connections */)) {
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
return;
}
}
}
ookla_exclude:
#endif
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct,
@@ -65,4 +61,3 @@ void init_ookla_dissector(struct ndpi_detection_module_struct *ndpi_struct,
*id += 1;
}
+4
View File
@@ -626,6 +626,10 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc
struct ndpi_packet_struct *packet = &flow->packet;
u_int8_t ret;
if(flow->packet.tcp && (5222 == ntohs(flow->packet.tcp->dest)))
printf("%u - %u\n", ntohs(flow->packet.tcp->source), ntohs(flow->packet.tcp->dest));
if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_SSL) {
if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) {
/* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets
-55
View File
@@ -1,55 +0,0 @@
#include <pthread.h>
#include <stdint.h>
#include <time.h>
#ifndef __lruc_header__
#define __lruc_header__
// ------------------------------------------
// errors
// ------------------------------------------
typedef enum {
LRUC_NO_ERROR = 0,
LRUC_MISSING_CACHE,
LRUC_MISSING_KEY,
LRUC_MISSING_VALUE,
LRUC_PTHREAD_ERROR,
LRUC_VALUE_TOO_LARGE
} lruc_error;
// ------------------------------------------
// types
// ------------------------------------------
typedef struct {
void *value;
void *key;
uint32_t value_length;
uint32_t key_length;
uint64_t access_count;
void *next;
} lruc_item;
typedef struct {
lruc_item **items;
uint64_t access_count;
uint64_t free_memory;
uint64_t total_memory;
uint64_t average_item_length;
uint32_t hash_table_size;
time_t seed;
lruc_item *free_items;
pthread_mutex_t *mutex;
} lruc;
// ------------------------------------------
// api
// ------------------------------------------
lruc *lruc_new(uint64_t cache_size, uint32_t average_length);
lruc_error lruc_free(lruc *cache);
lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length);
lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value);
lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length);
#endif
-294
View File
@@ -1,294 +0,0 @@
/* https://github.com/willcannings/C-LRU-Cache */
#include "lruc.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <err.h>
// ------------------------------------------
// private functions
// ------------------------------------------
// MurmurHash2, by Austin Appleby
// http://sites.google.com/site/murmurhash/
uint32_t lruc_hash(lruc *cache, void *key, uint32_t key_length) {
uint32_t m = 0x5bd1e995;
uint32_t r = 24;
uint32_t h = cache->seed ^ key_length;
char* data = (char *)key;
while(key_length >= 4) {
uint32_t k = *(uint32_t *)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
key_length -= 4;
}
switch(key_length) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
h *= m;
};
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return h % cache->hash_table_size;
}
// compare a key against an existing item's key
int lruc_cmp_keys(lruc_item *item, void *key, uint32_t key_length) {
if(key_length != item->key_length)
return 1;
else
return memcmp(key, item->key, key_length);
}
// remove an item and push it to the free items queue
void lruc_remove_item(lruc *cache, lruc_item *prev, lruc_item *item, uint32_t hash_index) {
if(prev)
prev->next = item->next;
else
cache->items[hash_index] = (lruc_item *) item->next;
// free memory and update the free memory counter
cache->free_memory += item->value_length;
free(item->value);
free(item->key);
// push the item to the free items queue
memset(item, 0, sizeof(lruc_item));
item->next = cache->free_items;
cache->free_items = item;
}
// remove the least recently used item
// TODO: we can optimise this by finding the n lru items, where n = required_space / average_length
void lruc_remove_lru_item(lruc *cache) {
lruc_item *min_item = NULL, *min_prev = NULL;
lruc_item *item = NULL, *prev = NULL;
uint32_t i = 0, min_index = -1;
uint64_t min_access_count = -1;
for(; i < cache->hash_table_size; i++) {
item = cache->items[i];
prev = NULL;
while(item) {
if(item->access_count < min_access_count || min_access_count == -1) {
min_access_count = item->access_count;
min_item = item;
min_prev = prev;
min_index = i;
}
prev = item;
item = item->next;
}
}
if(min_item)
lruc_remove_item(cache, min_prev, min_item, min_index);
}
// pop an existing item off the free queue, or create a new one
lruc_item *lruc_pop_or_create_item(lruc *cache) {
lruc_item *item = NULL;
if(cache->free_items) {
item = cache->free_items;
cache->free_items = item->next;
} else {
item = (lruc_item *) calloc(sizeof(lruc_item), 1);
}
return item;
}
// error helpers
#define error_for(conditions, error) if(conditions) {return error;}
#define test_for_missing_cache() error_for(!cache, LRUC_MISSING_CACHE)
#define test_for_missing_key() error_for(!key || key_length == 0, LRUC_MISSING_KEY)
#define test_for_missing_value() error_for(!value || value_length == 0, LRUC_MISSING_VALUE)
#define test_for_value_too_large() error_for(value_length > cache->total_memory, LRUC_VALUE_TOO_LARGE)
// lock helpers
#define lock_cache() if(pthread_mutex_lock(cache->mutex)) {\
perror("LRU Cache unable to obtain mutex lock");\
return LRUC_PTHREAD_ERROR;\
}
#define unlock_cache() if(pthread_mutex_unlock(cache->mutex)) {\
perror("LRU Cache unable to release mutex lock");\
return LRUC_PTHREAD_ERROR;\
}
// ------------------------------------------
// public api
// ------------------------------------------
lruc *lruc_new(uint64_t cache_size, uint32_t average_length) {
// create the cache
lruc *cache = (lruc *) calloc(sizeof(lruc), 1);
if(!cache) {
perror("LRU Cache unable to create cache object");
return NULL;
}
cache->hash_table_size = cache_size / average_length;
cache->average_item_length = average_length;
cache->free_memory = cache_size;
cache->total_memory = cache_size;
cache->seed = time(NULL);
// size the hash table to a guestimate of the number of slots required (assuming a perfect hash)
cache->items = (lruc_item **) calloc(sizeof(lruc_item *), cache->hash_table_size);
if(!cache->items) {
perror("LRU Cache unable to create cache hash table");
free(cache);
return NULL;
}
// all cache calls are guarded by a mutex
cache->mutex = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t));
if(pthread_mutex_init(cache->mutex, NULL)) {
perror("LRU Cache unable to initialise mutex");
free(cache->items);
free(cache);
return NULL;
}
return cache;
}
lruc_error lruc_free(lruc *cache) {
test_for_missing_cache();
// free each of the cached items, and the hash table
lruc_item *item = NULL, *next = NULL;
uint32_t i = 0;
if(cache->items) {
for(; i < cache->hash_table_size; i++) {
item = cache->items[i];
while(item) {
next = (lruc_item *) item->next;
free(item);
item = next;
}
}
free(cache->items);
}
// free the cache
if(cache->mutex) {
if(pthread_mutex_destroy(cache->mutex)) {
perror("LRU Cache unable to destroy mutex");
return LRUC_PTHREAD_ERROR;
}
}
free(cache);
return LRUC_NO_ERROR;
}
lruc_error lruc_set(lruc *cache, void *key, uint32_t key_length, void *value, uint32_t value_length) {
test_for_missing_cache();
test_for_missing_key();
test_for_missing_value();
test_for_value_too_large();
lock_cache();
// see if the key already exists
uint32_t hash_index = lruc_hash(cache, key, key_length), required = 0;
lruc_item *item = NULL, *prev = NULL;
item = cache->items[hash_index];
while(item && lruc_cmp_keys(item, key, key_length)) {
prev = item;
item = (lruc_item *) item->next;
}
if(item) {
// update the value and value_lengths
required = value_length - item->value_length;
free(item->value);
item->value = value;
item->value_length = value_length;
} else {
// insert a new item
item = lruc_pop_or_create_item(cache);
item->value = value;
item->key = key;
item->value_length = value_length;
item->key_length = key_length;
required = value_length;
if(prev)
prev->next = item;
else
cache->items[hash_index] = item;
}
item->access_count = ++cache->access_count;
// remove as many items as necessary to free enough space
if(required > 0 && required > cache->free_memory) {
while(cache->free_memory < required)
lruc_remove_lru_item(cache);
}
cache->free_memory -= required;
unlock_cache();
return LRUC_NO_ERROR;
}
lruc_error lruc_get(lruc *cache, void *key, uint32_t key_length, void **value) {
test_for_missing_cache();
test_for_missing_key();
lock_cache();
// loop until we find the item, or hit the end of a chain
uint32_t hash_index = lruc_hash(cache, key, key_length);
lruc_item *item = cache->items[hash_index];
while(item && lruc_cmp_keys(item, key, key_length))
item = (lruc_item *) item->next;
if(item) {
*value = item->value;
item->access_count = ++cache->access_count;
} else {
*value = NULL;
}
unlock_cache();
return LRUC_NO_ERROR;
}
lruc_error lruc_delete(lruc *cache, void *key, uint32_t key_length) {
test_for_missing_cache();
test_for_missing_key();
lock_cache();
// loop until we find the item, or hit the end of a chain
lruc_item *item = NULL, *prev = NULL;
uint32_t hash_index = lruc_hash(cache, key, key_length);
item = cache->items[hash_index];
while(item && lruc_cmp_keys(item, key, key_length)) {
prev = item;
item = (lruc_item *) item->next;
}
if(item) {
lruc_remove_item(cache, prev, item, hash_index);
}
unlock_cache();
return LRUC_NO_ERROR;
}