New version of the ahocorasick library (#1200)
The new version is about 25% faster with -O2 and 45% faster with -O3. No recursion is used (smaller stack size required). Uses less memory (by valgrind info) bigram: - original 1796 allocs, 247864 bytes allocated - new 1232 allocs, 158880 bytes allocated host_match: - original 18038 allocs, 3004576 bytes allocated - new 6861 allocs, 396624 bytes allocated The function ac_automata_search() is thread safe. Optional case-insensitive comparison. Matching at the beginning and at the end of the string is supported. One code file and one header file.
This commit is contained in:
+17
-1
@@ -2319,6 +2319,13 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
|
||||
ndpi_str->custom_categories.ipAddresses = ndpi_patricia_new(32 /* IPv4 */);
|
||||
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */);
|
||||
|
||||
if(ndpi_str->host_automa.ac_automa)
|
||||
ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC);
|
||||
if(ndpi_str->custom_categories.hostnames.ac_automa)
|
||||
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa,AC_FEATURE_LC);
|
||||
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
|
||||
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
|
||||
|
||||
if((ndpi_str->custom_categories.ipAddresses == NULL) || (ndpi_str->custom_categories.ipAddresses_shadow == NULL)) {
|
||||
NDPI_LOG_ERR(ndpi_str, "[NDPI] Error allocating Patricia trees\n");
|
||||
return(NULL);
|
||||
@@ -2441,6 +2448,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
|
||||
return(-2);
|
||||
|
||||
ac_input_text.astring = string_to_match, ac_input_text.length = strlen(string_to_match);
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(automa, &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
@@ -2470,6 +2478,7 @@ int ndpi_match_string_protocol_id(void *_automa, char *string_to_match,
|
||||
return(-2);
|
||||
|
||||
ac_input_text.astring = string_to_match, ac_input_text.length = match_len;
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(automa, &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
@@ -2503,6 +2512,7 @@ int ndpi_match_string_value(void *_automa, char *string_to_match,
|
||||
return(-2);
|
||||
|
||||
ac_input_text.astring = string_to_match, ac_input_text.length = match_len;
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(automa, &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
@@ -3020,8 +3030,11 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, con
|
||||
|
||||
static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char* domain_name) {
|
||||
if(ndpi_str->risky_domain_automa.ac_automa == NULL)
|
||||
if(ndpi_str->risky_domain_automa.ac_automa == NULL) {
|
||||
ndpi_str->risky_domain_automa.ac_automa = ac_automata_init(ac_match_handler);
|
||||
if(ndpi_str->risky_domain_automa.ac_automa)
|
||||
ac_automata_feature(ndpi_str->risky_domain_automa.ac_automa,AC_FEATURE_LC);
|
||||
}
|
||||
|
||||
if(ndpi_str->risky_domain_automa.ac_automa) {
|
||||
char buf[64], *str;
|
||||
@@ -6649,6 +6662,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_str,
|
||||
}
|
||||
|
||||
ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len;
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
@@ -6839,6 +6853,7 @@ int ndpi_match_bigram(struct ndpi_detection_module_struct *ndpi_str,
|
||||
}
|
||||
|
||||
ac_input_text.astring = bigram_to_match, ac_input_text.length = 2;
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
@@ -6873,6 +6888,7 @@ int ndpi_match_trigram(struct ndpi_detection_module_struct *ndpi_str,
|
||||
}
|
||||
|
||||
ac_input_text.astring = trigram_to_match, ac_input_text.length = 3;
|
||||
ac_input_text.ignore_case = 0;
|
||||
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
|
||||
|
||||
/*
|
||||
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
/*
|
||||
* actypes.h: Includes basic data types of ahocorasick library
|
||||
* This file is part of multifast.
|
||||
*
|
||||
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
|
||||
|
||||
multifast is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
multifast is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with multifast. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _AC_TYPES_H_
|
||||
#define _AC_TYPES_H_
|
||||
|
||||
/* AC_ALPHABET_t:
|
||||
* defines the alphabet type.
|
||||
* Actually defining AC_ALPHABET_t as a char will work, but sometimes we deal
|
||||
* with streams of other (bigger) types e.g. integers, specific enum, objects.
|
||||
* Although they consists of string of bytes (chars), but using their specific
|
||||
* types for AC_ALPHABET_t will lead to a better performance. so instead of
|
||||
* dealing with strings of chars, we assume dealing with strings of
|
||||
* AC_ALPHABET_t and leave it optional for other developers to define their
|
||||
* own alphabets.
|
||||
**/
|
||||
typedef char AC_ALPHABET_t;
|
||||
|
||||
/* AC_REP_t:
|
||||
* Provides a more readable representative for a pattern.
|
||||
* because patterns themselves are not always suitable for displaying
|
||||
* (e.g. for hex patterns), we offer this type to improve intelligibility
|
||||
* of output. furthermore, sometimes it is useful, for example while
|
||||
* retrieving patterns from a database, to maintain their identifiers in the
|
||||
* automata for further reference. we provisioned two possible types as a
|
||||
* union for this purpose. you can add your desired type in it.
|
||||
**/
|
||||
typedef struct {
|
||||
u_int32_t number; /* Often used to store procotolId */
|
||||
u_int16_t category, breed;
|
||||
} AC_REP_t;
|
||||
|
||||
/* AC_PATTERN_t:
|
||||
* This is the pattern type that must be fed into AC automata.
|
||||
* the 'astring' field is not null-terminated, due to it can contain zero
|
||||
* value bytes. the 'length' field determines the number of AC_ALPHABET_t it
|
||||
* carries. the 'representative' field is described in AC_REP_t. despite
|
||||
* 'astring', 'representative' can have duplicate values for different given
|
||||
* AC_PATTERN_t. it is an optional field and you can just fill it with 0.
|
||||
* CAUTION:
|
||||
* Not always the 'astring' points to the correct position in memory.
|
||||
* it is the responsibility of your program to maintain a permanent allocation
|
||||
* for astring field of the added pattern to automata.
|
||||
**/
|
||||
typedef struct
|
||||
{
|
||||
AC_ALPHABET_t * astring; /* String of alphabets */
|
||||
unsigned int length; /* Length of pattern */
|
||||
u_int8_t is_existing; /* 1 if the node is already part of another AC_PATTERN_t */
|
||||
AC_REP_t rep; /* Representative string (optional) */
|
||||
} AC_PATTERN_t;
|
||||
|
||||
/* AC_TEXT_t:
|
||||
* The input text type that is fed to ac_automata_search() to be searched.
|
||||
* it is similar to AC_PATTERN_t. actually we could use AC_PATTERN_t as input
|
||||
* text, but for the purpose of being more readable, we defined this new type.
|
||||
**/
|
||||
typedef struct
|
||||
{
|
||||
AC_ALPHABET_t * astring; /* String of alphabets */
|
||||
unsigned int length; /* Length of string */
|
||||
} AC_TEXT_t;
|
||||
|
||||
/* AC_MATCH_t:
|
||||
* Provides the structure for reporting a match event.
|
||||
* a match event occurs when the automata reaches a final node. any final
|
||||
* node can match one or more pattern at a position in a text. the
|
||||
* 'patterns' field holds these matched patterns. obviously these
|
||||
* matched patterns have same end-position in the text. there is a relationship
|
||||
* between matched patterns: the shorter one is a factor (tail) of the longer
|
||||
* one. the 'position' maintains the end position of matched patterns. the
|
||||
* start position of patterns could be found by knowing their 'length' in
|
||||
* AC_PATTERN_t. e.g. suppose "recent" and "cent" are matched at
|
||||
* position 40 in the text, then the start position of them are 34 and 36
|
||||
* respectively. finally the field 'match_num' maintains the number of
|
||||
* matched patterns.
|
||||
**/
|
||||
typedef struct
|
||||
{
|
||||
AC_PATTERN_t * patterns; /* Array of matched pattern */
|
||||
long position; /* The end position of matching pattern(s) in the text */
|
||||
unsigned int match_num; /* Number of matched patterns */
|
||||
} AC_MATCH_t;
|
||||
|
||||
/* AC_ERROR_t:
|
||||
* Error that may occur while adding a pattern to the automata.
|
||||
* it is returned by ac_automata_add().
|
||||
**/
|
||||
typedef enum
|
||||
{
|
||||
ACERR_SUCCESS = 0, /* No error occurred */
|
||||
ACERR_DUPLICATE_PATTERN, /* Duplicate patterns */
|
||||
ACERR_LONG_PATTERN, /* Pattern length is longer than AC_PATTRN_MAX_LENGTH */
|
||||
ACERR_ZERO_PATTERN, /* Empty pattern (zero length) */
|
||||
ACERR_AUTOMATA_CLOSED, /* Automata is closed. after calling
|
||||
ac_automata_finalize() you can not add new patterns to the automata. */
|
||||
} AC_ERROR_t;
|
||||
|
||||
/* MATCH_CALLBACK_t:
|
||||
* This is the call-back function type that must be given to automata at
|
||||
* initialization to report match occurrence to the caller.
|
||||
* at a match event, the automata will reach you using this function and sends
|
||||
* you a pointer to AC_MATCH_t. using that pointer you can handle
|
||||
* matches. you can send parameters to the call-back function when you call
|
||||
* ac_automata_search(). at call-back, the automata will sent you those
|
||||
* parameters as the second parameter (void *) of MATCH_CALLBACK_t. inside
|
||||
* the call-back function you can cast it to whatever you want.
|
||||
* If you return 0 from MATCH_CALLBACK_t function to the automata, it will
|
||||
* continue searching, otherwise it will return from ac_automata_search()
|
||||
* to your calling function.
|
||||
**/
|
||||
typedef int (*MATCH_CALLBACK_f)(AC_MATCH_t *, AC_TEXT_t *, AC_REP_t *);
|
||||
|
||||
/* AC_PATTRN_MAX_LENGTH:
|
||||
* Maximum acceptable pattern length in AC_PATTERN_t.length
|
||||
**/
|
||||
#define AC_PATTRN_MAX_LENGTH 1024
|
||||
|
||||
#endif
|
||||
+219
-27
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* ahocorasick.h: the main ahocorasick header file.
|
||||
* ahocorasick.h: Includes all types of ahocorasick library
|
||||
* This file is part of multifast.
|
||||
*
|
||||
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
|
||||
@@ -18,53 +18,245 @@
|
||||
along with multifast. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _AUTOMATA_H_
|
||||
#define _AUTOMATA_H_
|
||||
#ifndef _AC_TYPES_H_
|
||||
#define _AC_TYPES_H_
|
||||
|
||||
#include "node.h"
|
||||
#define AC_AHOCORASICK_NEW
|
||||
#define AC_PATTRN_MAX_LENGTH 256
|
||||
|
||||
/* reallocation step for AC_NODE_t.matched_patterns */
|
||||
#define REALLOC_CHUNK_MATCHSTR 8
|
||||
|
||||
/* reallocation step for AC_NODE_t.outgoing array */
|
||||
/* Must be a multiple of __SIZEOF_LONG__ ! */
|
||||
#define REALLOC_CHUNK_OUTGOING 8
|
||||
|
||||
/* AC_ALPHABET_t:
|
||||
* defines the alphabet type.
|
||||
* Actually defining AC_ALPHABET_t as a char will work, but sometimes we deal
|
||||
* with streams of other (bigger) types e.g. integers, specific enum, objects.
|
||||
* Although they consists of string of bytes (chars), but using their specific
|
||||
* types for AC_ALPHABET_t will lead to a better performance. so instead of
|
||||
* dealing with strings of chars, we assume dealing with strings of
|
||||
* AC_ALPHABET_t and leave it optional for other developers to define their
|
||||
* own alphabets.
|
||||
**/
|
||||
typedef char AC_ALPHABET_t;
|
||||
|
||||
/* AC_REP_t:
|
||||
* Provides a more readable representative for a pattern.
|
||||
* because patterns themselves are not always suitable for displaying
|
||||
* (e.g. for hex patterns), we offer this type to improve intelligibility
|
||||
* of output. furthermore, sometimes it is useful, for example while
|
||||
* retrieving patterns from a database, to maintain their identifiers in the
|
||||
* automata for further reference. we provisioned two possible types as a
|
||||
* union for this purpose. you can add your desired type in it.
|
||||
**/
|
||||
typedef struct {
|
||||
uint32_t number; /* Often used to store procotolId */
|
||||
uint16_t breed,
|
||||
category:14,from_start:1,at_end:1;
|
||||
} AC_REP_t;
|
||||
|
||||
/* AC_PATTERN_t:
|
||||
* This is the pattern type that must be fed into AC automata.
|
||||
* the 'astring' field is not null-terminated, due to it can contain zero
|
||||
* value bytes. the 'length' field determines the number of AC_ALPHABET_t it
|
||||
* carries. the 'representative' field is described in AC_REP_t. despite
|
||||
* 'astring', 'representative' can have duplicate values for different given
|
||||
* AC_PATTERN_t. it is an optional field and you can just fill it with 0.
|
||||
* CAUTION:
|
||||
* Not always the 'astring' points to the correct position in memory.
|
||||
* it is the responsibility of your program to maintain a permanent allocation
|
||||
* for astring field of the added pattern to automata.
|
||||
**/
|
||||
|
||||
typedef struct
|
||||
{
|
||||
AC_ALPHABET_t * astring; /* String of alphabets */
|
||||
uint16_t length, /* Length of pattern */
|
||||
is_existing; /* not union_matchstr */
|
||||
AC_REP_t rep; /* Representative string (optional) */
|
||||
} AC_PATTERN_t;
|
||||
|
||||
typedef struct {
|
||||
unsigned short num; /* Number of matched patterns at this node */
|
||||
unsigned short max; /* Max capacity of allocated memory for matched_patterns */
|
||||
AC_PATTERN_t patterns[];
|
||||
} AC_PATTERNS_t;
|
||||
|
||||
|
||||
/* AC_MATCH_t:
|
||||
* Provides the structure for reporting a match event.
|
||||
* a match event occurs when the automata reaches a final node. any final
|
||||
* node can match one or more pattern at a position in a text. the
|
||||
* 'patterns' field holds these matched patterns. obviously these
|
||||
* matched patterns have same end-position in the text. there is a relationship
|
||||
* between matched patterns: the shorter one is a factor (tail) of the longer
|
||||
* one. the 'position' maintains the end position of matched patterns. the
|
||||
* start position of patterns could be found by knowing their 'length' in
|
||||
* AC_PATTERN_t. e.g. suppose "recent" and "cent" are matched at
|
||||
* position 40 in the text, then the start position of them are 34 and 36
|
||||
* respectively. finally the field 'match_num' maintains the number of
|
||||
* matched patterns.
|
||||
**/
|
||||
|
||||
typedef struct
|
||||
{
|
||||
AC_PATTERN_t *matched[4]; /* for ac_automata_exact_match() */
|
||||
AC_PATTERN_t *patterns; /* Array of matched pattern */
|
||||
unsigned int position; /* The end position of matching pattern(s) in the text */
|
||||
unsigned short int match_num; /* Number of matched patterns */
|
||||
unsigned short int match_counter; /* Counter of found matches */
|
||||
} AC_MATCH_t;
|
||||
|
||||
/* AC_TEXT_t:
|
||||
* The input text type that is fed to ac_automata_search() to be searched.
|
||||
* it is similar to AC_PATTERN_t. actually we could use AC_PATTERN_t as input
|
||||
* text, but for the purpose of being more readable, we defined this new type.
|
||||
**/
|
||||
typedef struct
|
||||
{
|
||||
AC_MATCH_t match;
|
||||
AC_ALPHABET_t * astring; /* String of alphabets */
|
||||
unsigned short int length, /* Length of string */
|
||||
ignore_case;
|
||||
} AC_TEXT_t;
|
||||
|
||||
|
||||
/* AC_ERROR_t:
|
||||
* Error that may occur while adding a pattern to the automata.
|
||||
* it is returned by ac_automata_add().
|
||||
**/
|
||||
typedef enum
|
||||
{
|
||||
ACERR_SUCCESS = 0, /* No error occurred */
|
||||
ACERR_DUPLICATE_PATTERN, /* Duplicate patterns */
|
||||
ACERR_LONG_PATTERN, /* Pattern length is longer than AC_PATTRN_MAX_LENGTH */
|
||||
ACERR_ZERO_PATTERN, /* Empty pattern (zero length) */
|
||||
ACERR_AUTOMATA_CLOSED, /* Automata is closed. after calling
|
||||
ac_automata_finalize() you can not add new patterns to the automata. */
|
||||
ACERR_ERROR, /* common error */
|
||||
} AC_ERROR_t;
|
||||
|
||||
/* MATCH_CALLBACK_t:
|
||||
* This is the call-back function type that must be given to automata at
|
||||
* initialization to report match occurrence to the caller.
|
||||
* at a match event, the automata will reach you using this function and sends
|
||||
* you a pointer to AC_MATCH_t. using that pointer you can handle
|
||||
* matches. you can send parameters to the call-back function when you call
|
||||
* ac_automata_search(). at call-back, the automata will sent you those
|
||||
* parameters as the second parameter (void *) of MATCH_CALLBACK_t. inside
|
||||
* the call-back function you can cast it to whatever you want.
|
||||
* If you return 0 from MATCH_CALLBACK_t function to the automata, it will
|
||||
* continue searching, otherwise it will return from ac_automata_search()
|
||||
* to your calling function.
|
||||
**/
|
||||
typedef int (*MATCH_CALLBACK_f)(AC_MATCH_t *, AC_TEXT_t *, AC_REP_t *);
|
||||
|
||||
/* AC_PATTRN_MAX_LENGTH:
|
||||
* Maximum acceptable pattern length in AC_PATTERN_t.length
|
||||
**/
|
||||
|
||||
/* Forward Declaration */
|
||||
struct edge;
|
||||
|
||||
/*
|
||||
* automata node
|
||||
* 4 pointers + 8 bytes : 40/24 bytes for 64/32 bit
|
||||
*/
|
||||
typedef struct ac_node
|
||||
{
|
||||
int id; /* Node ID : set after finalize(), only for ac_automata_dump */
|
||||
AC_ALPHABET_t one_alpha,
|
||||
one:1, /* one_char: yes/no */
|
||||
range:1, /* range symbols start from one_alpha */
|
||||
root:1, /* is root node */
|
||||
final:1, /* 0: no ; 1: yes, it is a final node */
|
||||
use:1, /* use: yes/no */
|
||||
ff:1; /* finalized node */
|
||||
unsigned short depth; /* depth: distance between this node and the root */
|
||||
|
||||
AC_PATTERNS_t *matched_patterns; /* Array of matched patterns */
|
||||
struct edge *outgoing; /* Array of outgoing edges */
|
||||
|
||||
struct ac_node *failure_node; /* The failure node of this node */
|
||||
AC_ALPHABET_t *a_ptr;
|
||||
} AC_NODE_t;
|
||||
|
||||
#ifndef __SIZEOF_POINTER__
|
||||
#error SIZEOF_POINTER not defined!
|
||||
#endif
|
||||
|
||||
struct edge {
|
||||
unsigned short degree; /* Number of outgoing edges */
|
||||
unsigned short max; /* Max capacity of allocated memory for outgoing */
|
||||
uint32_t cmap[8]; /* 256 bit */
|
||||
AC_NODE_t *next[];
|
||||
/*
|
||||
* first N elements used for 'next' pointers +
|
||||
* M elements used for symbols storage
|
||||
* M = (max + sizeof(void*)-1) & ( ~(sizeof(void*)-1))
|
||||
*
|
||||
* if sizeof(void*)==8
|
||||
* for max = 8 we must alloc next[9];
|
||||
* for max = 16 we must alloc next[18];
|
||||
*
|
||||
*/
|
||||
};
|
||||
|
||||
struct ac_path {
|
||||
AC_NODE_t * n;
|
||||
unsigned short int idx,l;
|
||||
};
|
||||
|
||||
typedef struct
|
||||
{
|
||||
/* The root of the Aho-Corasick trie */
|
||||
AC_NODE_t * root;
|
||||
|
||||
/* maintain all nodes pointers. it will be used to access or release
|
||||
* all nodes. */
|
||||
AC_NODE_t ** all_nodes;
|
||||
|
||||
MATCH_CALLBACK_f match_handler;
|
||||
unsigned int all_nodes_num; /* Number of all nodes in the automata */
|
||||
unsigned int all_nodes_max; /* Current max allocated memory for *all_nodes */
|
||||
|
||||
AC_MATCH_t match; /* Any match is reported with this */
|
||||
MATCH_CALLBACK_f match_callback; /* Match call-back function */
|
||||
|
||||
/* this flag indicates that if automata is finalized by
|
||||
* ac_automata_finalize() or not. 1 means finalized and 0
|
||||
* means not finalized (is open). after finalizing automata you can not
|
||||
* add pattern to automata anymore. */
|
||||
unsigned short automata_open;
|
||||
unsigned short automata_open,
|
||||
to_lc:1, no_root_range:1; /* lowercase match */
|
||||
|
||||
/* Statistic Variables */
|
||||
unsigned long total_patterns; /* Total patterns in the automata */
|
||||
|
||||
unsigned long max_str_len; /* largest pattern length. Update by ac_automata_finalize() */
|
||||
|
||||
struct ac_path ac_path[AC_PATTRN_MAX_LENGTH+4]; /* for ac_automata_walk */
|
||||
int id; /* node id */
|
||||
int add_to_range; /* for convert to range */
|
||||
int n_oc,n_range,n_find; /* statistics */
|
||||
} AC_AUTOMATA_t;
|
||||
|
||||
typedef struct {
|
||||
/* It is possible to feed a large input to the automata chunk by chunk to
|
||||
* be searched using ac_automata_search(). in fact by default automata
|
||||
* thinks that all chunks are related unless you do ac_automata_reset().
|
||||
* followings are variables that keep track of searching state. */
|
||||
AC_NODE_t * current_node; /* Pointer to current node while searching */
|
||||
unsigned long base_position; /* Represents the position of current chunk
|
||||
related to whole input text */
|
||||
} AC_SEARCH_t;
|
||||
typedef AC_ERROR_t (*NODE_CALLBACK_f)(AC_AUTOMATA_t *, AC_NODE_t *,int idx, void *);
|
||||
|
||||
typedef void (*ALPHA_CALLBACK_f)(AC_AUTOMATA_t *, AC_NODE_t *,AC_NODE_t *,int ,void *);
|
||||
|
||||
#define AC_FEATURE_LC 1
|
||||
#define AC_FEATURE_NO_ROOT_RANGE 2
|
||||
|
||||
AC_AUTOMATA_t * ac_automata_init (MATCH_CALLBACK_f mc);
|
||||
AC_ERROR_t ac_automata_feature (AC_AUTOMATA_t * thiz, unsigned int feature);
|
||||
AC_ERROR_t ac_automata_add (AC_AUTOMATA_t * thiz, AC_PATTERN_t * str);
|
||||
void ac_automata_finalize (AC_AUTOMATA_t * thiz);
|
||||
int ac_automata_search (AC_AUTOMATA_t * thiz, AC_TEXT_t * str, AC_REP_t * param);
|
||||
void ac_automata_release (AC_AUTOMATA_t * thiz, u_int8_t free_pattern);
|
||||
void ac_automata_display (AC_AUTOMATA_t * thiz, char repcast);
|
||||
AC_ERROR_t ac_automata_finalize (AC_AUTOMATA_t * thiz);
|
||||
AC_ERROR_t ac_automata_walk (AC_AUTOMATA_t * thiz, NODE_CALLBACK_f node_cb,
|
||||
ALPHA_CALLBACK_f alpha_cb, void *);
|
||||
|
||||
int ac_automata_search (AC_AUTOMATA_t * thiz,
|
||||
AC_TEXT_t * str,
|
||||
AC_REP_t * param);
|
||||
int ac_automata_exact_match(AC_PATTERNS_t *mp,int pos, AC_TEXT_t *);
|
||||
void ac_automata_clean (AC_AUTOMATA_t * thiz);
|
||||
void ac_automata_release (AC_AUTOMATA_t * thiz, uint8_t free_pattern);
|
||||
#ifndef __KERNEL__
|
||||
void ac_automata_dump (AC_AUTOMATA_t * thiz,
|
||||
char *buf, size_t bufsize, char repcast);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
Vendored
-66
@@ -1,66 +0,0 @@
|
||||
/*
|
||||
* node.h: automata node header file
|
||||
* This file is part of multifast.
|
||||
*
|
||||
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
|
||||
|
||||
multifast is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
multifast is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with multifast. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _NODE_H_
|
||||
#define _NODE_H_
|
||||
|
||||
#include "actypes.h"
|
||||
|
||||
/* Forward Declaration */
|
||||
struct edge;
|
||||
|
||||
/* automata node */
|
||||
typedef struct ac_node
|
||||
{
|
||||
int id; /* Node ID : for debugging purpose */
|
||||
short int final; /* 0: no ; 1: yes, it is a final node */
|
||||
struct ac_node * failure_node; /* The failure node of this node */
|
||||
unsigned short depth; /* depth: distance between this node and the root */
|
||||
|
||||
/* Matched patterns */
|
||||
AC_PATTERN_t * matched_patterns; /* Array of matched patterns */
|
||||
unsigned short matched_patterns_num; /* Number of matched patterns at this node */
|
||||
unsigned short matched_patterns_max; /* Max capacity of allocated memory for matched_patterns */
|
||||
|
||||
/* Outgoing Edges */
|
||||
struct edge * outgoing; /* Array of outgoing edges */
|
||||
unsigned short outgoing_degree; /* Number of outgoing edges */
|
||||
unsigned short outgoing_max; /* Max capacity of allocated memory for outgoing */
|
||||
} AC_NODE_t;
|
||||
|
||||
/* The Edge of the Node */
|
||||
struct edge
|
||||
{
|
||||
AC_ALPHABET_t alpha; /* Edge alpha */
|
||||
struct ac_node * next; /* Target of the edge */
|
||||
};
|
||||
|
||||
|
||||
AC_NODE_t * node_create (void);
|
||||
AC_NODE_t * node_create_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
|
||||
void node_register_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * str, u_int8_t is_existing);
|
||||
void node_register_outgoing (AC_NODE_t * thiz, AC_NODE_t * next, AC_ALPHABET_t alpha);
|
||||
AC_NODE_t * node_find_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
|
||||
AC_NODE_t * node_findbs_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha);
|
||||
void node_release (AC_NODE_t * thiz, u_int8_t free_pattern);
|
||||
void node_assign_id (AC_NODE_t * thiz);
|
||||
void node_sort_edges (AC_NODE_t * thiz);
|
||||
|
||||
#endif
|
||||
Vendored
-6
@@ -1,6 +0,0 @@
|
||||
/* This is a function ported from the Linux kernel lib/sort.c */
|
||||
|
||||
void sort(void *base, size_t num, size_t len,
|
||||
int (*cmp_func)(const void *, const void *),
|
||||
void (*swap_func)(void *, void *, int size));
|
||||
|
||||
+1008
-190
File diff suppressed because it is too large
Load Diff
Vendored
-274
@@ -1,274 +0,0 @@
|
||||
/*
|
||||
* node.c: implementation of automata node
|
||||
* This file is part of multifast.
|
||||
*
|
||||
Copyright 2010-2012 Kamiar Kanani <kamiar.kanani@gmail.com>
|
||||
|
||||
multifast is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
multifast is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with multifast. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include "ndpi_api.h"
|
||||
#include "../include/node.h"
|
||||
#include "sort.h"
|
||||
|
||||
/* reallocation step for AC_NODE_t.matched_patterns */
|
||||
#define REALLOC_CHUNK_MATCHSTR 8
|
||||
|
||||
/* reallocation step for AC_NODE_t.outgoing array */
|
||||
#define REALLOC_CHUNK_OUTGOING 8
|
||||
/* TODO: For different depth of node, number of outgoing edges differs
|
||||
considerably, It is efficient to use different chunk size for
|
||||
different depths */
|
||||
|
||||
/* Private function prototype */
|
||||
void node_init (AC_NODE_t * thiz);
|
||||
int node_edge_compare (const void * l, const void * r);
|
||||
int node_has_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * newstr);
|
||||
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_create
|
||||
* Create the node
|
||||
******************************************************************************/
|
||||
AC_NODE_t * node_create(void)
|
||||
{
|
||||
AC_NODE_t * thiz = (AC_NODE_t *) ndpi_malloc (sizeof(AC_NODE_t));
|
||||
node_init(thiz);
|
||||
node_assign_id(thiz);
|
||||
return thiz;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_init
|
||||
* Initialize node
|
||||
******************************************************************************/
|
||||
void node_init(AC_NODE_t * thiz)
|
||||
{
|
||||
memset(thiz, 0, sizeof(AC_NODE_t));
|
||||
|
||||
thiz->outgoing_max = REALLOC_CHUNK_OUTGOING;
|
||||
thiz->outgoing = (struct edge *) ndpi_malloc
|
||||
(thiz->outgoing_max*sizeof(struct edge));
|
||||
|
||||
thiz->matched_patterns_max = REALLOC_CHUNK_MATCHSTR;
|
||||
thiz->matched_patterns = (AC_PATTERN_t *) ndpi_malloc
|
||||
(thiz->matched_patterns_max*sizeof(AC_PATTERN_t));
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_release
|
||||
* Release node
|
||||
******************************************************************************/
|
||||
void node_release(AC_NODE_t * thiz, u_int8_t free_pattern)
|
||||
{
|
||||
if(free_pattern) {
|
||||
for(int i=0; i<thiz->matched_patterns_num; i++) {
|
||||
if(!thiz->matched_patterns[i].is_existing)
|
||||
ndpi_free(thiz->matched_patterns[i].astring);
|
||||
}
|
||||
}
|
||||
|
||||
ndpi_free(thiz->matched_patterns);
|
||||
ndpi_free(thiz->outgoing);
|
||||
ndpi_free(thiz);
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_find_next
|
||||
* Find out the next node for a given Alpha to move. this function is used in
|
||||
* the pre-processing stage in which edge array is not sorted. so it uses
|
||||
* linear search.
|
||||
******************************************************************************/
|
||||
AC_NODE_t * node_find_next(AC_NODE_t * thiz, AC_ALPHABET_t alpha)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i=0; i < thiz->outgoing_degree; i++)
|
||||
{
|
||||
if(thiz->outgoing[i].alpha == alpha)
|
||||
return (thiz->outgoing[i].next);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_findbs_next
|
||||
* Find out the next node for a given Alpha. this function is used after the
|
||||
* pre-processing stage in which we sort edges. so it uses Binary Search.
|
||||
******************************************************************************/
|
||||
AC_NODE_t * node_findbs_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha)
|
||||
{
|
||||
int min, max, mid;
|
||||
AC_ALPHABET_t amid;
|
||||
|
||||
min = 0;
|
||||
max = thiz->outgoing_degree - 1;
|
||||
|
||||
while (min <= max)
|
||||
{
|
||||
mid = (min+max) >> 1;
|
||||
amid = thiz->outgoing[mid].alpha;
|
||||
if (alpha > amid)
|
||||
min = mid + 1;
|
||||
else if (alpha < amid)
|
||||
max = mid - 1;
|
||||
else
|
||||
return (thiz->outgoing[mid].next);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_has_matchstr
|
||||
* Determine if a final node contains a pattern in its accepted pattern list
|
||||
* or not. return values: 1 = it has, 0 = it hasn't
|
||||
******************************************************************************/
|
||||
int node_has_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * newstr)
|
||||
{
|
||||
int i;
|
||||
#if 0
|
||||
int j;
|
||||
#endif
|
||||
AC_PATTERN_t * str;
|
||||
|
||||
for (i=0; i < thiz->matched_patterns_num; i++)
|
||||
{
|
||||
str = &thiz->matched_patterns[i];
|
||||
|
||||
if (str->length != newstr->length)
|
||||
continue;
|
||||
|
||||
#if 0
|
||||
/* Original code */
|
||||
for (j=0; j<(int)str->length; j++)
|
||||
if(str->astring[j] != newstr->astring[j])
|
||||
continue;
|
||||
|
||||
if (j == str->length)
|
||||
return 1;
|
||||
#else
|
||||
/* https://github.com/ntop/nDPI/issues/837 */
|
||||
if (strncmp(str->astring, newstr->astring, str->length) == 0)
|
||||
return 1;
|
||||
#endif
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_create_next
|
||||
* Create the next node for the given alpha.
|
||||
******************************************************************************/
|
||||
AC_NODE_t * node_create_next (AC_NODE_t * thiz, AC_ALPHABET_t alpha)
|
||||
{
|
||||
AC_NODE_t * next;
|
||||
next = node_find_next (thiz, alpha);
|
||||
if (next)
|
||||
/* The edge already exists */
|
||||
return NULL;
|
||||
/* Otherwise register new edge */
|
||||
next = node_create ();
|
||||
node_register_outgoing(thiz, next, alpha);
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_register_matchstr
|
||||
* Adds the pattern to the list of accepted pattern.
|
||||
******************************************************************************/
|
||||
void node_register_matchstr (AC_NODE_t * thiz, AC_PATTERN_t * str, u_int8_t is_existing)
|
||||
{
|
||||
/* Check if the new pattern already exists in the node list */
|
||||
if (node_has_matchstr(thiz, str))
|
||||
return;
|
||||
|
||||
/* Manage memory */
|
||||
if (thiz->matched_patterns_num >= thiz->matched_patterns_max)
|
||||
{
|
||||
thiz->matched_patterns = (AC_PATTERN_t *) ndpi_realloc
|
||||
(thiz->matched_patterns, thiz->matched_patterns_max*sizeof(AC_PATTERN_t),
|
||||
(REALLOC_CHUNK_MATCHSTR+thiz->matched_patterns_max)*sizeof(AC_PATTERN_t));
|
||||
|
||||
thiz->matched_patterns_max += REALLOC_CHUNK_MATCHSTR;
|
||||
}
|
||||
|
||||
thiz->matched_patterns[thiz->matched_patterns_num].astring = str->astring;
|
||||
thiz->matched_patterns[thiz->matched_patterns_num].length = str->length;
|
||||
thiz->matched_patterns[thiz->matched_patterns_num].is_existing = is_existing;
|
||||
memcpy(&thiz->matched_patterns[thiz->matched_patterns_num].rep, &str->rep, sizeof(AC_REP_t));
|
||||
thiz->matched_patterns_num++;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_register_outgoing
|
||||
* Establish an edge between two nodes
|
||||
******************************************************************************/
|
||||
void node_register_outgoing
|
||||
(AC_NODE_t * thiz, AC_NODE_t * next, AC_ALPHABET_t alpha)
|
||||
{
|
||||
if(thiz->outgoing_degree >= thiz->outgoing_max)
|
||||
{
|
||||
thiz->outgoing = (struct edge *) ndpi_realloc
|
||||
(thiz->outgoing, thiz->outgoing_max*sizeof(struct edge),
|
||||
(REALLOC_CHUNK_OUTGOING+thiz->outgoing_max)*sizeof(struct edge));
|
||||
thiz->outgoing_max += REALLOC_CHUNK_OUTGOING;
|
||||
}
|
||||
|
||||
thiz->outgoing[thiz->outgoing_degree].alpha = alpha;
|
||||
thiz->outgoing[thiz->outgoing_degree++].next = next;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_assign_id
|
||||
* assign a unique ID to the node (used for debugging purpose).
|
||||
******************************************************************************/
|
||||
void node_assign_id (AC_NODE_t * thiz)
|
||||
{
|
||||
static int unique_id = 1;
|
||||
thiz->id = unique_id ++;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_edge_compare
|
||||
* Comparison function for qsort. see man qsort.
|
||||
******************************************************************************/
|
||||
int node_edge_compare (const void * l, const void * r)
|
||||
{
|
||||
/* According to man page:
|
||||
* The comparison function must return an integer less than, equal to, or
|
||||
* greater than zero if the first argument is considered to be
|
||||
* respectively less than, equal to, or greater than the second. if two
|
||||
* members compare as equal, their order in the sorted array is undefined.
|
||||
*
|
||||
* NOTE: Because edge alphabets are unique in every node we ignore
|
||||
* equivalence case.
|
||||
**/
|
||||
if ( ((struct edge *)l)->alpha >= ((struct edge *)r)->alpha )
|
||||
return 1;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* FUNCTION: node_sort_edges
|
||||
* sorts edges alphabets.
|
||||
******************************************************************************/
|
||||
void node_sort_edges (AC_NODE_t * thiz)
|
||||
{
|
||||
sort ((void *)thiz->outgoing, thiz->outgoing_degree, sizeof(struct edge), node_edge_compare, NULL);
|
||||
}
|
||||
Vendored
-92
@@ -1,92 +0,0 @@
|
||||
/*
|
||||
* A fast, small, non-recursive O(nlog n) sort for the Linux kernel
|
||||
*
|
||||
* Jan 23 2005 Matt Mackall <mpm@selenic.com>
|
||||
*/
|
||||
|
||||
#ifdef WIN32
|
||||
#include <stdint.h>
|
||||
typedef uint32_t u_int32_t;
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
/* This is a function ported from the Linux kernel lib/sort.c */
|
||||
|
||||
static void u_int32_t_swap(void *a, void *b, int size)
|
||||
{
|
||||
u_int32_t t = *(u_int32_t *)a;
|
||||
*(u_int32_t *)a = *(u_int32_t *)b;
|
||||
*(u_int32_t *)b = t;
|
||||
}
|
||||
|
||||
static void generic_swap(void *_a, void *_b, int size)
|
||||
{
|
||||
char t;
|
||||
char *a = (char*)_a;
|
||||
char *b = (char*)_b;
|
||||
|
||||
do {
|
||||
t = *a;
|
||||
*a++ = *b;
|
||||
*b++ = t;
|
||||
} while (--size > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* sort - sort an array of elements
|
||||
* @base: pointer to data to sort
|
||||
* @num: number of elements
|
||||
* @size: size of each element
|
||||
* @cmp_func: pointer to comparison function
|
||||
* @swap_func: pointer to swap function or NULL
|
||||
*
|
||||
* This function does a heapsort on the given array. You may provide a
|
||||
* swap_func function optimized to your element type.
|
||||
*
|
||||
* Sorting time is O(n log n) both on average and worst-case. While
|
||||
* qsort is about 20% faster on average, it suffers from exploitable
|
||||
* O(n*n) worst-case behavior and extra memory requirements that make
|
||||
* it less suitable for kernel use.
|
||||
*/
|
||||
|
||||
void sort(void *_base, size_t num, size_t size,
|
||||
int (*cmp_func)(const void *, const void *),
|
||||
void (*swap_func)(void *, void *, int size))
|
||||
{
|
||||
/* pre-scale counters for performance */
|
||||
int i = (num/2 - 1) * size, n = num * size, c, r;
|
||||
char *base = (char*)_base;
|
||||
|
||||
if (!swap_func)
|
||||
swap_func = (size == 4 ? u_int32_t_swap : generic_swap);
|
||||
|
||||
/* heapify */
|
||||
for ( ; i >= 0; i -= size) {
|
||||
for (r = i; r * 2 + size < n; r = c) {
|
||||
c = r * 2 + size;
|
||||
if (c < n - size &&
|
||||
cmp_func(base + c, base + c + size) < 0)
|
||||
c += size;
|
||||
if (cmp_func(base + r, base + c) >= 0)
|
||||
break;
|
||||
swap_func(base + r, base + c, size);
|
||||
}
|
||||
}
|
||||
|
||||
/* sort */
|
||||
for (i = n - size; i > 0; i -= size) {
|
||||
swap_func(base, base + i, size);
|
||||
for (r = 0; r * 2 + size < i; r = c) {
|
||||
c = r * 2 + size;
|
||||
if (c < i - size &&
|
||||
cmp_func(base + c, base + c + size) < 0)
|
||||
c += size;
|
||||
if (cmp_func(base + r, base + c) >= 0)
|
||||
break;
|
||||
swap_func(base + r, base + c, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user