New version of the ahocorasick library (#1200)

The new version is about 25% faster with -O2 and 45% faster with -O3.
No recursion is used (smaller stack size required).
Uses less memory (by valgrind info)
 bigram:
  - original 1796 allocs,   247864 bytes allocated
  - new      1232 allocs,   158880 bytes allocated
 host_match:
  - original 18038 allocs, 3004576 bytes allocated
  - new       6861 allocs,  396624 bytes allocated

The function ac_automata_search() is thread safe.
Optional case-insensitive comparison.
Matching at the beginning and at the end of the string is supported.
One code file and one header file.
This commit is contained in:
Vitaly Lavrov
2021-06-07 14:19:40 +02:00
committed by GitHub
parent 2fcf641e87
commit 65678dbeea
8 changed files with 1244 additions and 792 deletions
+17 -1
View File
@@ -2319,6 +2319,13 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->custom_categories.ipAddresses = ndpi_patricia_new(32 /* IPv4 */);
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */);
if(ndpi_str->host_automa.ac_automa)
ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
if((ndpi_str->custom_categories.ipAddresses == NULL) || (ndpi_str->custom_categories.ipAddresses_shadow == NULL)) {
NDPI_LOG_ERR(ndpi_str, "[NDPI] Error allocating Patricia trees\n");
return(NULL);
@@ -2441,6 +2448,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
return(-2);
ac_input_text.astring = string_to_match, ac_input_text.length = strlen(string_to_match);
ac_input_text.ignore_case = 0;
rc = ac_automata_search(automa, &ac_input_text, &match);
/*
@@ -2470,6 +2478,7 @@ int ndpi_match_string_protocol_id(void *_automa, char *string_to_match,
return(-2);
ac_input_text.astring = string_to_match, ac_input_text.length = match_len;
ac_input_text.ignore_case = 0;
rc = ac_automata_search(automa, &ac_input_text, &match);
/*
@@ -2503,6 +2512,7 @@ int ndpi_match_string_value(void *_automa, char *string_to_match,
return(-2);
ac_input_text.astring = string_to_match, ac_input_text.length = match_len;
ac_input_text.ignore_case = 0;
rc = ac_automata_search(automa, &ac_input_text, &match);
/*
@@ -3020,8 +3030,11 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, con
static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
char* domain_name) {
if(ndpi_str->risky_domain_automa.ac_automa == NULL)
if(ndpi_str->risky_domain_automa.ac_automa == NULL) {
ndpi_str->risky_domain_automa.ac_automa = ac_automata_init(ac_match_handler);
if(ndpi_str->risky_domain_automa.ac_automa)
ac_automata_feature(ndpi_str->risky_domain_automa.ac_automa,AC_FEATURE_LC);
}
if(ndpi_str->risky_domain_automa.ac_automa) {
char buf[64], *str;
@@ -6649,6 +6662,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_str,
}
ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len;
ac_input_text.ignore_case = 0;
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
/*
@@ -6839,6 +6853,7 @@ int ndpi_match_bigram(struct ndpi_detection_module_struct *ndpi_str,
}
ac_input_text.astring = bigram_to_match, ac_input_text.length = 2;
ac_input_text.ignore_case = 0;
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
/*
@@ -6873,6 +6888,7 @@ int ndpi_match_trigram(struct ndpi_detection_module_struct *ndpi_str,
}
ac_input_text.astring = trigram_to_match, ac_input_text.length = 3;
ac_input_text.ignore_case = 0;
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
/*
-136
View File
@@ -1,136 +0,0 @@
/*
* actypes.h: Includes basic data types of ahocorasick library
* This file is part of multifast.
*
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
multifast is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
multifast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with multifast. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _AC_TYPES_H_
#define _AC_TYPES_H_
/* AC_ALPHABET_t:
* defines the alphabet type.
* Actually defining AC_ALPHABET_t as a char will work, but sometimes we deal
* with streams of other (bigger) types e.g. integers, specific enum, objects.
* Although they consists of string of bytes (chars), but using their specific
* types for AC_ALPHABET_t will lead to a better performance. so instead of
* dealing with strings of chars, we assume dealing with strings of
* AC_ALPHABET_t and leave it optional for other developers to define their
* own alphabets.
**/
typedef char AC_ALPHABET_t;
/* AC_REP_t:
* Provides a more readable representative for a pattern.
* because patterns themselves are not always suitable for displaying
* (e.g. for hex patterns), we offer this type to improve intelligibility
* of output. furthermore, sometimes it is useful, for example while
* retrieving patterns from a database, to maintain their identifiers in the
* automata for further reference. we provisioned two possible types as a
* union for this purpose. you can add your desired type in it.
**/
typedef struct {
u_int32_t number; /* Often used to store procotolId */
u_int16_t category, breed;
} AC_REP_t;
/* AC_PATTERN_t:
* This is the pattern type that must be fed into AC automata.
* the 'astring' field is not null-terminated, due to it can contain zero
* value bytes. the 'length' field determines the number of AC_ALPHABET_t it
* carries. the 'representative' field is described in AC_REP_t. despite
* 'astring', 'representative' can have duplicate values for different given
* AC_PATTERN_t. it is an optional field and you can just fill it with 0.
* CAUTION:
* Not always the 'astring' points to the correct position in memory.
* it is the responsibility of your program to maintain a permanent allocation
* for astring field of the added pattern to automata.
**/
typedef struct
{
AC_ALPHABET_t * astring; /* String of alphabets */
unsigned int length; /* Length of pattern */
u_int8_t is_existing; /* 1 if the node is already part of another AC_PATTERN_t */
AC_REP_t rep; /* Representative string (optional) */
} AC_PATTERN_t;
/* AC_TEXT_t:
* The input text type that is fed to ac_automata_search() to be searched.
* it is similar to AC_PATTERN_t. actually we could use AC_PATTERN_t as input
* text, but for the purpose of being more readable, we defined this new type.
**/
typedef struct
{
AC_ALPHABET_t * astring; /* String of alphabets */
unsigned int length; /* Length of string */
} AC_TEXT_t;
/* AC_MATCH_t:
* Provides the structure for reporting a match event.
* a match event occurs when the automata reaches a final node. any final
* node can match one or more pattern at a position in a text. the
* 'patterns' field holds these matched patterns. obviously these
* matched patterns have same end-position in the text. there is a relationship
* between matched patterns: the shorter one is a factor (tail) of the longer
* one. the 'position' maintains the end position of matched patterns. the
* start position of patterns could be found by knowing their 'length' in
* AC_PATTERN_t. e.g. suppose "recent" and "cent" are matched at
* position 40 in the text, then the start position of them are 34 and 36
* respectively. finally the field 'match_num' maintains the number of
* matched patterns.
**/
typedef struct
{
AC_PATTERN_t * patterns; /* Array of matched pattern */
long position; /* The end position of matching pattern(s) in the text */
unsigned int match_num; /* Number of matched patterns */
} AC_MATCH_t;
/* AC_ERROR_t:
* Error that may occur while adding a pattern to the automata.
* it is returned by ac_automata_add().
**/
typedef enum
{
ACERR_SUCCESS = 0, /* No error occurred */
ACERR_DUPLICATE_PATTERN, /* Duplicate patterns */
ACERR_LONG_PATTERN, /* Pattern length is longer than AC_PATTRN_MAX_LENGTH */
ACERR_ZERO_PATTERN, /* Empty pattern (zero length) */
ACERR_AUTOMATA_CLOSED, /* Automata is closed. after calling
ac_automata_finalize() you can not add new patterns to the automata. */
} AC_ERROR_t;
/* MATCH_CALLBACK_t:
* This is the call-back function type that must be given to automata at
* initialization to report match occurrence to the caller.
* at a match event, the automata will reach you using this function and sends
* you a pointer to AC_MATCH_t. using that pointer you can handle
* matches. you can send parameters to the call-back function when you call
* ac_automata_search(). at call-back, the automata will sent you those
* parameters as the second parameter (void *) of MATCH_CALLBACK_t. inside
* the call-back function you can cast it to whatever you want.
* If you return 0 from MATCH_CALLBACK_t function to the automata, it will
* continue searching, otherwise it will return from ac_automata_search()
* to your calling function.
**/
typedef int (*MATCH_CALLBACK_f)(AC_MATCH_t *, AC_TEXT_t *, AC_REP_t *);
/* AC_PATTRN_MAX_LENGTH:
* Maximum acceptable pattern length in AC_PATTERN_t.length
**/
#define AC_PATTRN_MAX_LENGTH 1024
#endif
+219 -27
View File
@@ -1,5 +1,5 @@
/*
* ahocorasick.h: the main ahocorasick header file.
* ahocorasick.h: Includes all types of ahocorasick library
* This file is part of multifast.
*
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
@@ -18,53 +18,245 @@
along with multifast. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _AUTOMATA_H_
#define _AUTOMATA_H_
#ifndef _AC_TYPES_H_
#define _AC_TYPES_H_
#include "node.h"
#define AC_AHOCORASICK_NEW
#define AC_PATTRN_MAX_LENGTH 256
/* reallocation step for AC_NODE_t.matched_patterns */
#define REALLOC_CHUNK_MATCHSTR 8
/* reallocation step for AC_NODE_t.outgoing array */
/* Must be a multiple of __SIZEOF_LONG__ ! */
#define REALLOC_CHUNK_OUTGOING 8
/* AC_ALPHABET_t:
* defines the alphabet type.
* Actually defining AC_ALPHABET_t as a char will work, but sometimes we deal
* with streams of other (bigger) types e.g. integers, specific enum, objects.
* Although they consists of string of bytes (chars), but using their specific
* types for AC_ALPHABET_t will lead to a better performance. so instead of
* dealing with strings of chars, we assume dealing with strings of
* AC_ALPHABET_t and leave it optional for other developers to define their
* own alphabets.
**/
typedef char AC_ALPHABET_t;
/* AC_REP_t:
* Provides a more readable representative for a pattern.
* because patterns themselves are not always suitable for displaying
* (e.g. for hex patterns), we offer this type to improve intelligibility
* of output. furthermore, sometimes it is useful, for example while
* retrieving patterns from a database, to maintain their identifiers in the
* automata for further reference. we provisioned two possible types as a
* union for this purpose. you can add your desired type in it.
**/
typedef struct {
uint32_t number; /* Often used to store procotolId */
uint16_t breed,
category:14,from_start:1,at_end:1;
} AC_REP_t;
/* AC_PATTERN_t:
* This is the pattern type that must be fed into AC automata.
* the 'astring' field is not null-terminated, due to it can contain zero
* value bytes. the 'length' field determines the number of AC_ALPHABET_t it
* carries. the 'representative' field is described in AC_REP_t. despite
* 'astring', 'representative' can have duplicate values for different given
* AC_PATTERN_t. it is an optional field and you can just fill it with 0.
* CAUTION:
* Not always the 'astring' points to the correct position in memory.
* it is the responsibility of your program to maintain a permanent allocation
* for astring field of the added pattern to automata.
**/
typedef struct
{
AC_ALPHABET_t * astring; /* String of alphabets */
uint16_t length, /* Length of pattern */
is_existing; /* not union_matchstr */
AC_REP_t rep; /* Representative string (optional) */
} AC_PATTERN_t;
typedef struct {
unsigned short num; /* Number of matched patterns at this node */
unsigned short max; /* Max capacity of allocated memory for matched_patterns */
AC_PATTERN_t patterns[];
} AC_PATTERNS_t;
/* AC_MATCH_t:
* Provides the structure for reporting a match event.
* a match event occurs when the automata reaches a final node. any final
* node can match one or more pattern at a position in a text. the
* 'patterns' field holds these matched patterns. obviously these
* matched patterns have same end-position in the text. there is a relationship
* between matched patterns: the shorter one is a factor (tail) of the longer
* one. the 'position' maintains the end position of matched patterns. the
* start position of patterns could be found by knowing their 'length' in
* AC_PATTERN_t. e.g. suppose "recent" and "cent" are matched at
* position 40 in the text, then the start position of them are 34 and 36
* respectively. finally the field 'match_num' maintains the number of
* matched patterns.
**/
typedef struct
{
AC_PATTERN_t *matched[4]; /* for ac_automata_exact_match() */
AC_PATTERN_t *patterns; /* Array of matched pattern */
unsigned int position; /* The end position of matching pattern(s) in the text */
unsigned short int match_num; /* Number of matched patterns */
unsigned short int match_counter; /* Counter of found matches */
} AC_MATCH_t;
/* AC_TEXT_t:
* The input text type that is fed to ac_automata_search() to be searched.
* it is similar to AC_PATTERN_t. actually we could use AC_PATTERN_t as input
* text, but for the purpose of being more readable, we defined this new type.
**/
typedef struct
{
AC_MATCH_t match;
AC_ALPHABET_t * astring; /* String of alphabets */
unsigned short int length, /* Length of string */
ignore_case;
} AC_TEXT_t;
/* AC_ERROR_t:
* Error that may occur while adding a pattern to the automata.
* it is returned by ac_automata_add().
**/
typedef enum
{
ACERR_SUCCESS = 0, /* No error occurred */
ACERR_DUPLICATE_PATTERN, /* Duplicate patterns */
ACERR_LONG_PATTERN, /* Pattern length is longer than AC_PATTRN_MAX_LENGTH */
ACERR_ZERO_PATTERN, /* Empty pattern (zero length) */
ACERR_AUTOMATA_CLOSED, /* Automata is closed. after calling
ac_automata_finalize() you can not add new patterns to the automata. */
ACERR_ERROR, /* common error */
} AC_ERROR_t;
/* MATCH_CALLBACK_t:
* This is the call-back function type that must be given to automata at
* initialization to report match occurrence to the caller.
* at a match event, the automata will reach you using this function and sends
* you a pointer to AC_MATCH_t. using that pointer you can handle
* matches. you can send parameters to the call-back function when you call
* ac_automata_search(). at call-back, the automata will sent you those
* parameters as the second parameter (void *) of MATCH_CALLBACK_t. inside
* the call-back function you can cast it to whatever you want.
* If you return 0 from MATCH_CALLBACK_t function to the automata, it will
* continue searching, otherwise it will return from ac_automata_search()
* to your calling function.
**/
typedef int (*MATCH_CALLBACK_f)(AC_MATCH_t *, AC_TEXT_t *, AC_REP_t *);
/* AC_PATTRN_MAX_LENGTH:
* Maximum acceptable pattern length in AC_PATTERN_t.length
**/
/* Forward Declaration */
struct edge;
/*
* automata node
* 4 pointers + 8 bytes : 40/24 bytes for 64/32 bit
*/
typedef struct ac_node
{
int id; /* Node ID : set after finalize(), only for ac_automata_dump */
AC_ALPHABET_t one_alpha,
one:1, /* one_char: yes/no */
range:1, /* range symbols start from one_alpha */
root:1, /* is root node */
final:1, /* 0: no ; 1: yes, it is a final node */
use:1, /* use: yes/no */
ff:1; /* finalized node */
unsigned short depth; /* depth: distance between this node and the root */
AC_PATTERNS_t *matched_patterns; /* Array of matched patterns */
struct edge *outgoing; /* Array of outgoing edges */
struct ac_node *failure_node; /* The failure node of this node */
AC_ALPHABET_t *a_ptr;
} AC_NODE_t;
#ifndef __SIZEOF_POINTER__
#error SIZEOF_POINTER not defined!
#endif
struct edge {
unsigned short degree; /* Number of outgoing edges */
unsigned short max; /* Max capacity of allocated memory for outgoing */
uint32_t cmap[8]; /* 256 bit */
AC_NODE_t *next[];
/*
* first N elements used for 'next' pointers +
* M elements used for symbols storage
* M = (max + sizeof(void*)-1) & ( ~(sizeof(void*)-1))
*
* if sizeof(void*)==8
* for max = 8 we must alloc next[9];
* for max = 16 we must alloc next[18];
*
*/
};
struct ac_path {
AC_NODE_t * n;
unsigned short int idx,l;
};
typedef struct
{
/* The root of the Aho-Corasick trie */
AC_NODE_t * root;
/* maintain all nodes pointers. it will be used to access or release
* all nodes. */
AC_NODE_t ** all_nodes;
MATCH_CALLBACK_f match_handler;
unsigned int all_nodes_num; /* Number of all nodes in the automata */
unsigned int all_nodes_max; /* Current max allocated memory for *all_nodes */
AC_MATCH_t match; /* Any match is reported with this */
MATCH_CALLBACK_f match_callback; /* Match call-back function */
/* this flag indicates that if automata is finalized by
* ac_automata_finalize() or not. 1 means finalized and 0
* means not finalized (is open). after finalizing automata you can not
* add pattern to automata anymore. */
unsigned short automata_open;
unsigned short automata_open,
to_lc:1, no_root_range:1; /* lowercase match */
/* Statistic Variables */
unsigned long total_patterns; /* Total patterns in the automata */
unsigned long max_str_len; /* largest pattern length. Update by ac_automata_finalize() */
struct ac_path ac_path[AC_PATTRN_MAX_LENGTH+4]; /* for ac_automata_walk */
int id; /* node id */
int add_to_range; /* for convert to range */
int n_oc,n_range,n_find; /* statistics */
} AC_AUTOMATA_t;
typedef struct {
/* It is possible to feed a large input to the automata chunk by chunk to
* be searched using ac_automata_search(). in fact by default automata
* thinks that all chunks are related unless you do ac_automata_reset().
* followings are variables that keep track of searching state. */
AC_NODE_t * current_node; /* Pointer to current node while searching */
unsigned long base_position; /* Represents the position of current chunk
related to whole input text */
} AC_SEARCH_t;
typedef AC_ERROR_t (*NODE_CALLBACK_f)(AC_AUTOMATA_t *, AC_NODE_t *,int idx, void *);
typedef void (*ALPHA_CALLBACK_f)(AC_AUTOMATA_t *, AC_NODE_t *,AC_NODE_t *,int ,void *);
#define AC_FEATURE_LC 1
#define AC_FEATURE_NO_ROOT_RANGE 2
AC_AUTOMATA_t * ac_automata_init (MATCH_CALLBACK_f mc);
AC_ERROR_t ac_automata_feature (AC_AUTOMATA_t * thiz, unsigned int feature);
AC_ERROR_t ac_automata_add (AC_AUTOMATA_t * thiz, AC_PATTERN_t * str);
void ac_automata_finalize (AC_AUTOMATA_t * thiz);
int ac_automata_search (AC_AUTOMATA_t * thiz, AC_TEXT_t * str, AC_REP_t * param);
void ac_automata_release (AC_AUTOMATA_t * thiz, u_int8_t free_pattern);
void ac_automata_display (AC_AUTOMATA_t * thiz, char repcast);
AC_ERROR_t ac_automata_finalize (AC_AUTOMATA_t * thiz);
AC_ERROR_t ac_automata_walk (AC_AUTOMATA_t * thiz, NODE_CALLBACK_f node_cb,
ALPHA_CALLBACK_f alpha_cb, void *);
int ac_automata_search (AC_AUTOMATA_t * thiz,
AC_TEXT_t * str,
AC_REP_t * param);
int ac_automata_exact_match(AC_PATTERNS_t *mp,int pos, AC_TEXT_t *);
void ac_automata_clean (AC_AUTOMATA_t * thiz);
void ac_automata_release (AC_AUTOMATA_t * thiz, uint8_t free_pattern);
#ifndef __KERNEL__
void ac_automata_dump (AC_AUTOMATA_t * thiz,
char *buf, size_t bufsize, char repcast);
#endif
#endif
-66
View File
@@ -1,66 +0,0 @@
/*
* node.h: automata node header file
* This file is part of multifast.
*
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
multifast is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
multifast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with multifast. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _NODE_H_
#define _NODE_H_
#include "actypes.h"
/* Forward Declaration */
struct edge;
/* automata node */
typedef struct ac_node
{
int id; /* Node ID : for debugging purpose */
short int final; /* 0: no ; 1: yes, it is a final node */
struct ac_node * failure_node; /* The failure node of this node */
unsigned short depth; /* depth: distance between this node and the root */
/* Matched patterns */
AC_PATTERN_t * matched_patterns; /* Array of matched patterns */
unsigned short matched_patterns_num; /* Number of matched patterns at this node */
unsigned short matched_patterns_max; /* Max capacity of allocated memory for matched_patterns */
/* Outgoing Edges */
struct edge * outgoing; /* Array of outgoing edges */
unsigned short outgoing_degree; /* Number of outgoing edges */
unsigned short outgoing_max; /* Max capacity of allocated memory for outgoing */
} AC_NODE_t;
/* The Edge of the Node */
struct edge
{
AC_ALPHABET_t alpha; /* Edge alpha */
struct ac_node * next; /* Target of the edge */
};
AC_NODE_t * node_create (void);
AC_NODE_t * node_create_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
void node_register_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * str, u_int8_t is_existing);
void node_register_outgoing (AC_NODE_t * thiz, AC_NODE_t * next, AC_ALPHABET_t alpha);
AC_NODE_t * node_find_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
AC_NODE_t * node_findbs_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
void node_release (AC_NODE_t * thiz, u_int8_t free_pattern);
void node_assign_id (AC_NODE_t * thiz);
void node_sort_edges (AC_NODE_t * thiz);
#endif
-6
View File
@@ -1,6 +0,0 @@
/* This is a function ported from the Linux kernel lib/sort.c */
void sort(void *base, size_t num, size_t len,
int (*cmp_func)(const void *, const void *),
void (*swap_func)(void *, void *, int size));
+1008 -190
View File
File diff suppressed because it is too large Load Diff
-274
View File
@@ -1,274 +0,0 @@
/*
* node.c: implementation of automata node
* This file is part of multifast.
*
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
multifast is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
multifast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with multifast. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "ndpi_api.h"
#include "../include/node.h"
#include "sort.h"
/* reallocation step for AC_NODE_t.matched_patterns */
#define REALLOC_CHUNK_MATCHSTR 8
/* reallocation step for AC_NODE_t.outgoing array */
#define REALLOC_CHUNK_OUTGOING 8
/* TODO: For different depth of node, number of outgoing edges differs
considerably, It is efficient to use different chunk size for
different depths */
/* Private function prototype */
void node_init (AC_NODE_t * thiz);
int node_edge_compare (const void * l, const void * r);
int node_has_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * newstr);
/******************************************************************************
* FUNCTION: node_create
* Create the node
******************************************************************************/
AC_NODE_t * node_create(void)
{
AC_NODE_t * thiz = (AC_NODE_t *) ndpi_malloc (sizeof(AC_NODE_t));
node_init(thiz);
node_assign_id(thiz);
return thiz;
}
/******************************************************************************
* FUNCTION: node_init
* Initialize node
******************************************************************************/
void node_init(AC_NODE_t * thiz)
{
memset(thiz, 0, sizeof(AC_NODE_t));
thiz->outgoing_max = REALLOC_CHUNK_OUTGOING;
thiz->outgoing = (struct edge *) ndpi_malloc
(thiz->outgoing_max*sizeof(struct edge));
thiz->matched_patterns_max = REALLOC_CHUNK_MATCHSTR;
thiz->matched_patterns = (AC_PATTERN_t *) ndpi_malloc
(thiz->matched_patterns_max*sizeof(AC_PATTERN_t));
}
/******************************************************************************
* FUNCTION: node_release
* Release node
******************************************************************************/
void node_release(AC_NODE_t * thiz, u_int8_t free_pattern)
{
if(free_pattern) {
for(int i=0; i<thiz->matched_patterns_num; i++) {
if(!thiz->matched_patterns[i].is_existing)
ndpi_free(thiz->matched_patterns[i].astring);
}
}
ndpi_free(thiz->matched_patterns);
ndpi_free(thiz->outgoing);
ndpi_free(thiz);
}
/******************************************************************************
* FUNCTION: node_find_next
* Find out the next node for a given Alpha to move. this function is used in
* the pre-processing stage in which edge array is not sorted. so it uses
* linear search.
******************************************************************************/
AC_NODE_t * node_find_next(AC_NODE_t * thiz, AC_ALPHABET_t alpha)
{
int i;
for (i=0; i < thiz->outgoing_degree; i++)
{
if(thiz->outgoing[i].alpha == alpha)
return (thiz->outgoing[i].next);
}
return NULL;
}
/******************************************************************************
* FUNCTION: node_findbs_next
* Find out the next node for a given Alpha. this function is used after the
* pre-processing stage in which we sort edges. so it uses Binary Search.
******************************************************************************/
AC_NODE_t * node_findbs_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha)
{
int min, max, mid;
AC_ALPHABET_t amid;
min = 0;
max = thiz->outgoing_degree - 1;
while (min <= max)
{
mid = (min+max) >> 1;
amid = thiz->outgoing[mid].alpha;
if (alpha > amid)
min = mid + 1;
else if (alpha < amid)
max = mid - 1;
else
return (thiz->outgoing[mid].next);
}
return NULL;
}
/******************************************************************************
* FUNCTION: node_has_matchstr
* Determine if a final node contains a pattern in its accepted pattern list
* or not. return values: 1 = it has, 0 = it hasn't
******************************************************************************/
int node_has_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * newstr)
{
int i;
#if 0
int j;
#endif
AC_PATTERN_t * str;
for (i=0; i < thiz->matched_patterns_num; i++)
{
str = &thiz->matched_patterns[i];
if (str->length != newstr->length)
continue;
#if 0
/* Original code */
for (j=0; j<(int)str->length; j++)
if(str->astring[j] != newstr->astring[j])
continue;
if (j == str->length)
return 1;
#else
/* https://github.com/ntop/nDPI/issues/837 */
if (strncmp(str->astring, newstr->astring, str->length) == 0)
return 1;
#endif
}
return 0;
}
/******************************************************************************
* FUNCTION: node_create_next
* Create the next node for the given alpha.
******************************************************************************/
AC_NODE_t * node_create_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha)
{
AC_NODE_t * next;
next = node_find_next (thiz, alpha);
if (next)
/* The edge already exists */
return NULL;
/* Otherwise register new edge */
next = node_create ();
node_register_outgoing(thiz, next, alpha);
return next;
}
/******************************************************************************
* FUNCTION: node_register_matchstr
* Adds the pattern to the list of accepted pattern.
******************************************************************************/
void node_register_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * str, u_int8_t is_existing)
{
/* Check if the new pattern already exists in the node list */
if (node_has_matchstr(thiz, str))
return;
/* Manage memory */
if (thiz->matched_patterns_num >= thiz->matched_patterns_max)
{
thiz->matched_patterns = (AC_PATTERN_t *) ndpi_realloc
(thiz->matched_patterns, thiz->matched_patterns_max*sizeof(AC_PATTERN_t),
(REALLOC_CHUNK_MATCHSTR+thiz->matched_patterns_max)*sizeof(AC_PATTERN_t));
thiz->matched_patterns_max += REALLOC_CHUNK_MATCHSTR;
}
thiz->matched_patterns[thiz->matched_patterns_num].astring = str->astring;
thiz->matched_patterns[thiz->matched_patterns_num].length = str->length;
thiz->matched_patterns[thiz->matched_patterns_num].is_existing = is_existing;
memcpy(&thiz->matched_patterns[thiz->matched_patterns_num].rep, &str->rep, sizeof(AC_REP_t));
thiz->matched_patterns_num++;
}
/******************************************************************************
* FUNCTION: node_register_outgoing
* Establish an edge between two nodes
******************************************************************************/
void node_register_outgoing
(AC_NODE_t * thiz, AC_NODE_t * next, AC_ALPHABET_t alpha)
{
if(thiz->outgoing_degree >= thiz->outgoing_max)
{
thiz->outgoing = (struct edge *) ndpi_realloc
(thiz->outgoing, thiz->outgoing_max*sizeof(struct edge),
(REALLOC_CHUNK_OUTGOING+thiz->outgoing_max)*sizeof(struct edge));
thiz->outgoing_max += REALLOC_CHUNK_OUTGOING;
}
thiz->outgoing[thiz->outgoing_degree].alpha = alpha;
thiz->outgoing[thiz->outgoing_degree++].next = next;
}
/******************************************************************************
* FUNCTION: node_assign_id
* assign a unique ID to the node (used for debugging purpose).
******************************************************************************/
void node_assign_id (AC_NODE_t * thiz)
{
static int unique_id = 1;
thiz->id = unique_id ++;
}
/******************************************************************************
* FUNCTION: node_edge_compare
* Comparison function for qsort. see man qsort.
******************************************************************************/
int node_edge_compare (const void * l, const void * r)
{
/* According to man page:
* The comparison function must return an integer less than, equal to, or
* greater than zero if the first argument is considered to be
* respectively less than, equal to, or greater than the second. if two
* members compare as equal, their order in the sorted array is undefined.
*
* NOTE: Because edge alphabets are unique in every node we ignore
* equivalence case.
**/
if ( ((struct edge *)l)->alpha >= ((struct edge *)r)->alpha )
return 1;
else
return -1;
}
/******************************************************************************
* FUNCTION: node_sort_edges
* sorts edges alphabets.
******************************************************************************/
void node_sort_edges (AC_NODE_t * thiz)
{
sort ((void *)thiz->outgoing, thiz->outgoing_degree, sizeof(struct edge), node_edge_compare, NULL);
}
-92
View File
@@ -1,92 +0,0 @@
/*
* A fast, small, non-recursive O(nlog n) sort for the Linux kernel
*
* Jan 23 2005 Matt Mackall <mpm@selenic.com>
*/
#ifdef WIN32
#include <stdint.h>
typedef uint32_t u_int32_t;
#endif
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
/* This is a function ported from the Linux kernel lib/sort.c */
static void u_int32_t_swap(void *a, void *b, int size)
{
u_int32_t t = *(u_int32_t *)a;
*(u_int32_t *)a = *(u_int32_t *)b;
*(u_int32_t *)b = t;
}
static void generic_swap(void *_a, void *_b, int size)
{
char t;
char *a = (char*)_a;
char *b = (char*)_b;
do {
t = *a;
*a++ = *b;
*b++ = t;
} while (--size > 0);
}
/**
* sort - sort an array of elements
* @base: pointer to data to sort
* @num: number of elements
* @size: size of each element
* @cmp_func: pointer to comparison function
* @swap_func: pointer to swap function or NULL
*
* This function does a heapsort on the given array. You may provide a
* swap_func function optimized to your element type.
*
* Sorting time is O(n log n) both on average and worst-case. While
* qsort is about 20% faster on average, it suffers from exploitable
* O(n*n) worst-case behavior and extra memory requirements that make
* it less suitable for kernel use.
*/
void sort(void *_base, size_t num, size_t size,
int (*cmp_func)(const void *, const void *),
void (*swap_func)(void *, void *, int size))
{
/* pre-scale counters for performance */
int i = (num/2 - 1) * size, n = num * size, c, r;
char *base = (char*)_base;
if (!swap_func)
swap_func = (size == 4 ? u_int32_t_swap : generic_swap);
/* heapify */
for ( ; i >= 0; i -= size) {
for (r = i; r * 2 + size < n; r = c) {
c = r * 2 + size;
if (c < n - size &&
cmp_func(base + c, base + c + size) < 0)
c += size;
if (cmp_func(base + r, base + c) >= 0)
break;
swap_func(base + r, base + c, size);
}
}
/* sort */
for (i = n - size; i > 0; i -= size) {
swap_func(base, base + i, size);
for (r = 0; r * 2 + size < i; r = c) {
c = r * 2 + size;
if (c < i - size &&
cmp_func(base + c, base + c + size) < 0)
c += size;
if (cmp_func(base + r, base + c) >= 0)
break;
swap_func(base + r, base + c, size);
}
}
}