LRU caches: add a generic (optional and configurable) expiration logic (#1855)

Two caches already implemented a similar mechanism: make it generic.
This commit is contained in:
Ivan Nardi
2023-01-18 18:18:36 +01:00
committed by GitHub
parent de24206adc
commit 1b98bec0ab
12 changed files with 196 additions and 63 deletions
+5 -4
View File
@@ -123,19 +123,20 @@ static void ndpi_add_connection_as_bittorrent(struct ndpi_detection_module_struc
key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, flow->c_port), key2 = ndpi_ip_port_hash_funct(flow->s_address.v4, flow->s_port);
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key2, NDPI_PROTOCOL_BITTORRENT);
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key2, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
/* Now add hosts as twins */
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache,
flow->c_address.v4 + flow->s_address.v4,
NDPI_PROTOCOL_BITTORRENT);
NDPI_PROTOCOL_BITTORRENT,
ndpi_get_current_time(flow));
/* Also add +2 ports of the sender in order to catch additional sockets open by the same client */
for(i=0; i<2; i++) {
key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, htons(ntohs(flow->c_port)+1+i));
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
}
#ifdef BITTORRENT_CACHE_DEBUG
+1 -1
View File
@@ -80,7 +80,7 @@ void ndpi_search_hangout(struct ndpi_detection_module_struct *ndpi_struct,
printf("[LRU] ADDING %u / %u.%u\n", key, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_HANGOUT_DUO);
#endif
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_HANGOUT_DUO);
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_HANGOUT_DUO, ndpi_get_current_time(flow));
}
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO,
+3 -3
View File
@@ -1255,9 +1255,9 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
if(ndpi_struct->ookla_cache != NULL) {
if(packet->iph != NULL) {
if(packet->tcp->source == htons(8080))
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
else
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
} else if(packet->iphv6 != NULL) {
u_int32_t h;
@@ -1266,7 +1266,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
else
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
}
}
+1 -1
View File
@@ -44,7 +44,7 @@ u_int32_t make_mining_key(struct ndpi_flow_struct *flow) {
static void cacheMiningHostTwins(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
if(ndpi_struct->mining_cache)
ndpi_lru_add_to_cache(ndpi_struct->mining_cache, make_mining_key(flow), NDPI_PROTOCOL_MINING);
ndpi_lru_add_to_cache(ndpi_struct->mining_cache, make_mining_key(flow), NDPI_PROTOCOL_MINING, ndpi_get_current_time(flow));
}
/* ************************************************************************** */
+5 -3
View File
@@ -63,7 +63,7 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
#ifdef OOKLA_DEBUG
printf("=>>>>>>>> [OOKLA IPv6] Adding %u\n", h);
#endif
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
}
return;
} else {
@@ -79,7 +79,8 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
printf("=>>>>>>>> [OOKLA IPv6] Searching %u\n", h);
#endif
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */)) {
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
#ifdef OOKLA_DEBUG
@@ -110,7 +111,8 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
if(ndpi_struct->ookla_cache != NULL) {
u_int16_t dummy;
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */)) {
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
#ifdef OOKLA_DEBUG
+16 -15
View File
@@ -56,7 +56,7 @@ u_int32_t get_stun_lru_key(struct ndpi_flow_struct *flow, u_int8_t rev) {
int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
{
u_int16_t when;
u_int16_t dummy;
u_int32_t key;
if(ndpi_struct->stun_zoom_cache &&
@@ -67,15 +67,12 @@ int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct
#endif
if(ndpi_lru_find_cache(ndpi_struct->stun_zoom_cache, key,
&when, 0 /* Don't remove it as it can be used for other connections */)) {
u_int16_t tdiff = ((flow->last_packet_time_ms /1000) & 0xFFFF) - when;
&dummy, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
#ifdef DEBUG_ZOOM_LRU
printf("[LRU ZOOM] Found, diff %d\n", tdiff);
printf("[LRU ZOOM] Found");
#endif
if(tdiff < 60 /* sec */)
return 1;
return 1;
}
}
return 0;
@@ -102,7 +99,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
u_int16_t cached_proto;
if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key,
&cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
&cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
#ifdef DEBUG_LRU
printf("[LRU] FOUND %u / %u: no need to cache %u.%u\n", key, cached_proto, proto, app_proto);
#endif
@@ -114,7 +112,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
u_int32_t key_rev = get_stun_lru_key(flow, 1);
if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key_rev,
&cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
&cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
#ifdef DEBUG_LRU
printf("[LRU] FOUND %u / %u: no need to cache %u.%u\n", key_rev, cached_proto, proto, app_proto);
#endif
@@ -131,8 +130,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
ntohs(packet->udp->source), ntohs(packet->udp->dest));
#endif
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, app_proto);
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key_rev, app_proto);
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, app_proto, ndpi_get_current_time(flow));
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key_rev, app_proto, ndpi_get_current_time(flow));
}
}
}
@@ -147,7 +146,7 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
printf("[LRU ZOOM] ADDING %u [src_port %u]\n", key, ntohs(flow->c_port));
#endif
ndpi_lru_add_to_cache(ndpi_struct->stun_zoom_cache, key,
(flow->last_packet_time_ms / 1000) & 0xFFFF /* 16 bit */);
0 /* dummy */, ndpi_get_current_time(flow));
}
ndpi_set_detected_protocol(ndpi_struct, flow, app_proto, NDPI_PROTOCOL_STUN, confidence);
@@ -234,7 +233,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
u_int16_t proto;
u_int32_t key = get_stun_lru_key(flow, 0);
int rc = ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &proto,
0 /* Don't remove it as it can be used for other connections */);
0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow));
#ifdef DEBUG_LRU
printf("[LRU] Searching %u\n", key);
@@ -243,7 +243,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(!rc) {
key = get_stun_lru_key(flow, 1);
rc = ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &proto,
0 /* Don't remove it as it can be used for other connections */);
0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow));
#ifdef DEBUG_LRU
printf("[LRU] Searching %u\n", key);
+3 -2
View File
@@ -340,7 +340,8 @@ static void checkTLSSubprotocol(struct ndpi_detection_module_struct *ndpi_struct
key = make_tls_cert_key(packet, is_from_client);
if(ndpi_lru_find_cache(ndpi_struct->tls_cert_cache, key,
&cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
&cached_proto, 0 /* Don't remove it as it can be used for other connections */,
ndpi_get_current_time(flow))) {
ndpi_protocol ret = { __get_master(ndpi_struct, flow), cached_proto, NDPI_PROTOCOL_UNKNOWN /* unused */, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_set_detected_protocol(ndpi_struct, flow, cached_proto, __get_master(ndpi_struct, flow), NDPI_CONFIDENCE_DPI_CACHE);
@@ -730,7 +731,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
if(ndpi_struct->tls_cert_cache) {
u_int32_t key = make_tls_cert_key(packet, 0 /* from the server */);
ndpi_lru_add_to_cache(ndpi_struct->tls_cert_cache, key, proto_id);
ndpi_lru_add_to_cache(ndpi_struct->tls_cert_cache, key, proto_id, ndpi_get_current_time(flow));
}
}
}