Improved helper scripts. (#1986)
* added additional (more restrictive) checks Signed-off-by: lns <matzeton@googlemail.com>
This commit is contained in:
@@ -87,6 +87,8 @@ LT_LIB_M
|
||||
PKG_PROG_PKG_CONFIG
|
||||
|
||||
AC_PROG_CC
|
||||
AC_PROG_CPP_WERROR
|
||||
AC_C_INLINE
|
||||
|
||||
SYSTEM=`uname -s`
|
||||
if test $SYSTEM = "Darwin"; then
|
||||
|
||||
+5
-2
@@ -1,4 +1,6 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
FAILED_ASN=0
|
||||
TOTAL_ASN=0
|
||||
@@ -28,6 +30,7 @@ function create_list() {
|
||||
}
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
echo "(1) Downloading Apple routes..."
|
||||
DEST="../src/lib/inc_generated/ndpi_asn_apple.c.inc"
|
||||
@@ -194,7 +197,7 @@ DEST=../src/lib/inc_generated/ndpi_asn_nvidia.c.inc
|
||||
create_list NDPI_PROTOCOL_NVIDIA $DEST "AS60977" "AS50889" "AS20347" "AS11414"
|
||||
echo "(3) Nvidia IPs are available in $DEST"
|
||||
|
||||
if [ ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
|
||||
if [ ${TOTAL_ASN} -eq 0 -o ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
|
||||
printf '%s: %s\n' "${0}" "All download(s) failed, ./get_routes_by_asn.sh broken?"
|
||||
exit 1
|
||||
else
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_amazon_aws_match.c.inc
|
||||
TMP=/tmp/aws.json
|
||||
@@ -10,15 +13,15 @@ ORIGIN=https://ip-ranges.amazonaws.com/ip-ranges.json
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST # TODO: ipv6
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_AMAZON_AWS > $DEST
|
||||
rm -f $TMP $LIST
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) Amazon AWS IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_azure_match.c.inc
|
||||
LINK_TMP=/tmp/azure_link.txt
|
||||
@@ -14,30 +15,25 @@ LINK_ORIGIN="https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
|
||||
|
||||
echo "(1) Downloading file... ${LINK_ORIGIN}"
|
||||
http_response=$(curl -s -o ${LINK_TMP} -w "%{http_code}" ${LINK_ORIGIN})
|
||||
if [ "${http_response}" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the link origin url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LINK_TMP}"
|
||||
|
||||
ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href=\"[^"]+' | sed 's/href="//' | uniq)"
|
||||
ORIGIN="$(grep -E 'ServiceTags_Public_[[:digit:]]+.json' ${LINK_TMP} | grep -o -E 'href="[^"]+' | sed 's/href="//' | uniq)"
|
||||
rm -f ${LINK_TMP}
|
||||
if [ -z "${ORIGIN}" ]; then
|
||||
echo "Error ${LINK_ORIGIN} does not contain the url format!"
|
||||
exit 1
|
||||
fi
|
||||
is_str_empty "${ORIGIN}" "${LINK_ORIGIN} does not contain the url format!"
|
||||
|
||||
echo "(2) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ "${http_response}" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(3) Processing IP addresses..."
|
||||
# Note: the last "grep -v :" is used to skip IPv6 addresses
|
||||
tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_AZURE > $DEST
|
||||
rm -f $TMP $LIST
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(4) Microsoft Azure IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# List all the current bittorrent nodes
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
# NOTE: JQ can be found at https://stedolan.github.io/jq/
|
||||
|
||||
curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1
|
||||
RESULT="$(curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1)"
|
||||
is_str_empty "${RESULT}" "String empty, please review this script."
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_cachefly_match.c.inc
|
||||
LIST=/tmp/cachefly.list
|
||||
@@ -9,14 +12,13 @@ ORIGIN='https://cachefly.cachefly.net/ips/cdn.txt'
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
|
||||
if [ "${http_response}" != "200" ]; then
|
||||
echo "Error ${http_response}: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LIST}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_CACHEFLY > "${DEST}"
|
||||
rm -f "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) Cachefly IPs are available in ${DEST}"
|
||||
exit 0
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_cloudflare_match.c.inc
|
||||
LIST=/tmp/cloudflare.list
|
||||
@@ -10,14 +13,13 @@ ORIGIN="https://www.cloudflare.com/ips-v4"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LIST}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_CLOUDFLARE > $DEST
|
||||
rm -f $LIST
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) Cloudflare IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
printf 'Running script: %s\n' "$(basename ${0})" >&2
|
||||
|
||||
function check_http_response()
|
||||
{
|
||||
http_response="${1}"
|
||||
|
||||
if [ "${http_response}" != "200" ]; then
|
||||
printf '%s error: %s\n' "${0}" "HTTP Response code ${http_response}; you probably need to update the list url!" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function is_file_empty()
|
||||
{
|
||||
file="${1}"
|
||||
|
||||
if [ ! -r "${file}" ]; then
|
||||
printf '%s error: %s\n' "${0}" "file ${file} not found or not readable!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ `cat "${file}" | wc -l` -eq 0 ]; then
|
||||
printf '%s error: %s\n' "${0}" "file ${file} empty!" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function is_str_empty()
|
||||
{
|
||||
str="${1}"
|
||||
errmsg="${2}"
|
||||
|
||||
if [ -z "${str}" ]; then
|
||||
printf '%s error: %s\n' "${0}" "${errmsg}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_crawlers_match.c.inc
|
||||
TMP1=/tmp/bot_google_c1.json
|
||||
@@ -23,34 +24,27 @@ ORIGIN_BING="https://www.bing.com/toolbox/bingbot.json"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN1}"
|
||||
http_response=$(curl -s -o $TMP1 -w "%{http_code}" ${ORIGIN1})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP1}"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN2}"
|
||||
http_response=$(curl -s -o $TMP2 -w "%{http_code}" ${ORIGIN2})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP2}"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN3}"
|
||||
http_response=$(curl -s -o $TMP3 -w "%{http_code}" ${ORIGIN3})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP3}"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN_BING}"
|
||||
http_response=$(curl -s -o $TMP_BING -w "%{http_code}" ${ORIGIN_BING})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP_BING}"
|
||||
|
||||
echo "(1) Downloading FB crawlers routes... "
|
||||
whois -h whois.radb.net -- '-i origin AS32934' | grep ^route > $TMP_FB
|
||||
is_file_empty "${TMP_FB}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
{
|
||||
@@ -60,7 +54,9 @@ echo "(2) Processing IP addresses..."
|
||||
jq -r '.prefixes | .[].ipv4Prefix | select( . != null )' $TMP_BING # TODO: ipv6
|
||||
grep -v route6 $TMP_FB | tr -d 'route:^ ' # TODO: ipv6
|
||||
} > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_HTTP_CRAWLER_BOT > $DEST
|
||||
is_file_empty "${DEST}"
|
||||
rm -f $TMP1 $TMP2 $TMP3 $TMP_BING $TMP_FB $LIST
|
||||
|
||||
echo "(3) Crawlers IPs are available in $DEST"
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_ethereum_match.c.inc
|
||||
TMP=/tmp/ethereum
|
||||
@@ -10,16 +13,16 @@ ORIGIN="https://raw.githubusercontent.com/ethereum/go-ethereum/master/params/boo
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1 > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MINING > $DEST
|
||||
rm -f $TMP $LIST
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) Ethereum/Mining IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
#!/usr/bin/env sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc
|
||||
LIST=/tmp/gambling.list
|
||||
|
||||
printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)"
|
||||
DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)"
|
||||
is_str_empty "${DOMAINS}" "Please check gambling sites URL and sed REGEX."
|
||||
|
||||
printf '(2) %s\n' "Processing IP addresses..."
|
||||
echo "${DOMAINS}" >${LIST}
|
||||
./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST}
|
||||
rm -f "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc
|
||||
TMP=/tmp/google_c.json
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_google_match.c.inc
|
||||
LIST=/tmp/google.list
|
||||
@@ -13,8 +14,10 @@ echo "(1) Downloading file..."
|
||||
echo "(2) Processing IP addresses..."
|
||||
#https://cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults
|
||||
python3 google.py > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE > $DEST
|
||||
#rm -f $TMP $LIST
|
||||
rm -f "${TMP}" "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) Google IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -39,11 +39,13 @@ print("""/*
|
||||
|
||||
print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {")
|
||||
|
||||
lines = 0
|
||||
with open(sys.argv[1]) as fp:
|
||||
for cnt, line in enumerate(fp):
|
||||
line = line.rstrip()
|
||||
|
||||
if(line != ""):
|
||||
lines += 1
|
||||
x = line.split("/")
|
||||
|
||||
if(len(x) == 2):
|
||||
@@ -57,3 +59,7 @@ with open(sys.argv[1]) as fp:
|
||||
print(" /* End */")
|
||||
print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }")
|
||||
print("};")
|
||||
|
||||
if lines == 0:
|
||||
sys.stderr.write(f'File {sys.argv[1]} is empty.\n')
|
||||
sys.exit(1)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_icloud_private_relay_match.c.inc
|
||||
TMP=/tmp/icloud.csv
|
||||
@@ -13,18 +14,19 @@ ORIGIN="https://mask-api.icloud.com/egress-ip-ranges.csv"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
cut -d ',' -f 1 $TMP | grep -v ':' > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./mergeipaddrlist.py $LIST > $LIST_MERGED
|
||||
is_file_empty "${LIST_MERGED}"
|
||||
./ipaddr2list.py $LIST_MERGED NDPI_ANONYMOUS_SUBSCRIBER > $DEST
|
||||
rm -f $TMP $LIST $LIST_MERGED
|
||||
is_file_empty "${DEST}"
|
||||
rm -f "${TMP}" "${LIST}" "${LIST_MERGED}"
|
||||
|
||||
echo "(3) iCloud Private Relay IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -40,11 +40,13 @@ print("""/*
|
||||
|
||||
print("static ndpi_network "+proto.lower()+"_protocol_list[] = {")
|
||||
|
||||
lines = 0
|
||||
with open(sys.argv[1]) as fp:
|
||||
for cnt, line in enumerate(fp):
|
||||
line = line.rstrip()
|
||||
|
||||
if(line != ""):
|
||||
lines += 1
|
||||
x = line.split("/")
|
||||
|
||||
if(len(x) == 2):
|
||||
@@ -60,3 +62,6 @@ with open(sys.argv[1]) as fp:
|
||||
print(" /* End */")
|
||||
print(" { 0x0, 0, 0 }")
|
||||
print("};")
|
||||
|
||||
if lines == 0:
|
||||
sys.stderr.write(f'{sys.argv[0]}: File {sys.argv[1]} is empty.\n')
|
||||
|
||||
@@ -16,5 +16,11 @@ ipFile = open(sys.argv[1])
|
||||
ipAddresses = list(ipFile.readlines())
|
||||
ipAddresses = sorted(ipAddresses)
|
||||
cidrs = netaddr.cidr_merge(ipAddresses)
|
||||
|
||||
lines = 0
|
||||
for cidr in cidrs:
|
||||
lines += 1
|
||||
print(cidr)
|
||||
|
||||
if lines == 0:
|
||||
sys.stderr.write(f'{sys.argv[0]}: file {sys.argv[1]} is empty\n')
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST_OUTLOOK=../src/lib/inc_generated/ndpi_ms_outlook_match.c.inc
|
||||
DEST_SKYPE_MSTEAMS=../src/lib/inc_generated/ndpi_ms_skype_teams_match.c.inc
|
||||
@@ -16,35 +17,41 @@ ORIGIN="https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
|
||||
#OUTLOOK
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Exchange") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_OUTLOOK > $DEST_OUTLOOK
|
||||
is_file_empty "${DEST_OUTLOOK}"
|
||||
|
||||
#SKYPE/TEAMS
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Skype") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_SKYPE_TEAMS > $DEST_SKYPE_MSTEAMS
|
||||
is_file_empty "${DEST_SKYPE_MSTEAMS}"
|
||||
|
||||
#ONEDRIVE
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="SharePoint") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_ONE_DRIVE > $DEST_ONEDRIVE
|
||||
is_file_empty "${DEST_ONEDRIVE}"
|
||||
|
||||
#OFFICE
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
#TODO: NDPI_PROTOCOL_MICROSOFT_365 or NDPI_PROTOCOL_MICROSOFT?
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_365 > $DEST_OFFICE365
|
||||
is_file_empty "${DEST_OFFICE365}"
|
||||
|
||||
rm -f $TMP $LIST
|
||||
rm -f "${TMP}" "${LIST}"
|
||||
|
||||
echo "(3) Microsoft IPs are available in $DEST_OUTLOOK, $DEST_SKYPE_MSTEAMS, $DEST_ONEDRIVE, $DEST_OFFICE365"
|
||||
echo "(3) Microsoft IPs are available in ${DEST_OUTLOOK}, ${DEST_SKYPE_MSTEAMS}, ${DEST_ONEDRIVE}, ${DEST_OFFICE365}"
|
||||
exit 0
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
|
||||
LIST=/tmp/tor.list
|
||||
@@ -13,14 +16,13 @@ ORIGIN="https://check.torproject.org/torbulkexitlist"
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LIST}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST
|
||||
rm -f $LIST
|
||||
rm -f "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) TOR IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc
|
||||
TMP=/tmp/wa.zip
|
||||
@@ -12,18 +13,19 @@ IP_LINK_URL='https://developers.facebook.com/docs/whatsapp/guides/network-requir
|
||||
|
||||
echo "(1) Scraping Facebook WhatsApp IP Adresses and Ranges..."
|
||||
ORIGIN="$(curl -s "${IP_LINK_URL}" | sed -ne 's/.*<a href="\([^"]*\)" target="_blank">WhatsApp server IP addresses and ranges (.zip file)<\/a>.*/\1/gp' | sed -e 's/\&/\&/g')"
|
||||
is_str_empty "${ORIGIN}" "IP webpage list does not contain any addresses. A REGEX update may be required."
|
||||
|
||||
echo "(2) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
http_response=$(curl -s -o "${TMP}" -w "%{http_code}" "${ORIGIN}")
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${TMP}"
|
||||
|
||||
echo "(3) Processing IP addresses..."
|
||||
unzip -p /tmp/wa.zip "WhatsApp IPs (IPv4 Only) 2022-07-26 - 2022-07-30.txt" > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_WHATSAPP > $DEST
|
||||
rm -f $TMP $LIST
|
||||
unzip -p /tmp/wa.zip "WhatsApp IPs (IPv4 Only) 2022-07-26 - 2022-07-30.txt" > "${LIST}"
|
||||
is_file_empty "${LIST}"
|
||||
./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_WHATSAPP > "${DEST}"
|
||||
rm -f "${TMP}" "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(4) WhatsApp IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "${0}")" || exit 1
|
||||
. ./common.sh || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_zoom_match.c.inc
|
||||
LIST=/tmp/zoom.list
|
||||
@@ -10,15 +13,14 @@ ORIGIN="https://assets.zoom.us/docs/ipranges/Zoom.txt"
|
||||
|
||||
|
||||
echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
exit 1
|
||||
fi
|
||||
http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LIST}"
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_ZOOM > $DEST
|
||||
rm -f $LIST
|
||||
./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_ZOOM > "${DEST}"
|
||||
rm -f "${LIST}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) ZOOM IPs are available in $DEST"
|
||||
echo "(3) ZOOM IPs are available in ${DEST}"
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user