Added scripts to auto generate hostname/SNI *.inc files. (#1984)

* add illegal gambling sites (Belgium)

Signed-off-by: lns <matzeton@googlemail.com>
This commit is contained in:
Toni
2023-05-20 15:41:15 +02:00
committed by GitHub
parent 9004d5c2ca
commit c97e2d67ff
8 changed files with 540 additions and 4 deletions
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env sh
set -e
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc
LIST=/tmp/gambling.list
printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)"
DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)"
echo "${DOMAINS}" >${LIST}
./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST}
rm -f "${LIST}"
exit 0
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
import sys
import socket, struct
# This scripts is used to create "hostname/sni -> protocols" lists.
if len(sys.argv) < 6:
print("Usage: {} <file> <name> <protocol> <category> <breed>".format(sys.argv[0]))
sys.exit (1)
name = sys.argv[2]
proto = sys.argv[3]
category = sys.argv[4]
breed = sys.argv[5]
print("""/*
*
* This file is generated automatically and part of nDPI
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
/* ****************************************************** */
""")
print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {")
with open(sys.argv[1]) as fp:
for cnt, line in enumerate(fp):
line = line.rstrip()
if(line != ""):
x = line.split("/")
if(len(x) == 2):
host = x[0]
else:
host = line
if(host != ""):
print(' { ' + f'"{host}", "{name}", {proto}, {category}, {breed}, NDPI_PROTOCOL_DEFAULT_LEVEL' + ' },')
print(" /* End */")
print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }")
print("};")
+3
View File
@@ -34,5 +34,8 @@ RETVAL=$(( RETVAL + $? ))
./asn_update.sh
RETVAL=$(( RETVAL + $? ))
./gambling_sites_download.sh
RETVAL=$(( RETVAL + $? ))
test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} script(s) failed"
exit ${RETVAL}