Added Edgecast and Cachefly CDNs. (#1540)

* Improved ASN update script
 * Ran `utils/update_every_lists.sh'
 * `tests/do.sh.in' prints the amount of failed pcap(s)
 * `utils/asn_update.sh' prints the amount of failed download(s)

Signed-off-by: lns <matzeton@googlemail.com>
This commit is contained in:
Toni
2022-05-07 09:33:25 +02:00
committed by GitHub
parent 87f93ea4fd
commit b2648a45a3
31 changed files with 2033 additions and 1385 deletions
+19 -2
View File
@@ -1,6 +1,7 @@
#!/bin/bash
set -e
FAILED_ASN=0
TOTAL_ASN=0
function processing_list() {
local LIST_MERGED="/tmp/list_m"
@@ -15,7 +16,11 @@ function create_list() {
LIST=/tmp/list
for i in "${@:3}"; do
./get_routes_by_asn.sh "$i" >> $LIST
TOTAL_ASN=$(( TOTAL_ASN + 1 ))
if ! ./get_routes_by_asn.sh "$i" >> $LIST; then
echo "Could not fetch route for ${i} (${1})"
FAILED_ASN=$(( FAILED_ASN + 1 ))
fi
done
processing_list "$LIST" "$1" "$2"
@@ -113,3 +118,15 @@ echo "(1) Downloading Citrix routes..." #Citrix or a more generic LogMeIn stuff?
DEST=../src/lib/inc_generated/ndpi_asn_citrix.c.inc
create_list NDPI_PROTOCOL_CITRIX $DEST "AS395424" "AS21866" "AS213380" "AS20104" "AS16815"
echo "(3) Citrix IPs are available in $DEST"
echo "(1) Downloading Edgecast routes..."
DEST=../src/lib/inc_generated/ndpi_asn_edgecast.c.inc
create_list NDPI_PROTOCOL_EDGECAST $DEST "AS15133"
echo "(3) Edgecast IPs are available in $DEST"
if [ ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
printf '%s: %s\n' "${0}" "All download(s) failed, ./get_routes_by_asn.sh broken?"
exit 1
else
exit 0
fi
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
cd "$(dirname "${0}")"
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_amazon_aws_match.c.inc
TMP=/tmp/aws.json
+3 -1
View File
@@ -1,6 +1,8 @@
#!/bin/sh
cd "$(dirname "${0}")"
set -e
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_azure_match.c.inc
LINK_TMP=/tmp/azure_link.txt
+2
View File
@@ -3,6 +3,8 @@
# List all the current bittorrent nodes
#
set -e
# NOTE: JQ can be found at https://stedolan.github.io/jq/
curl -s -H "Accept: application/json; indent=4" https://bitnodes.io/api/v1/snapshots/latest/ | jq -r '.nodes|keys[] as $k | "\($k)"' | grep -v onion | grep -v ']' | cut -d ':' -f 1
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_cachefly_match.c.inc
LIST=/tmp/cachefly.list
ORIGIN='https://cachefly.cachefly.net/ips/cdn.txt'
echo "(1) Downloading file..."
http_response=$(curl -s -o "${LIST}" -w "%{http_code}" "${ORIGIN}")
if [ "${http_response}" != "200" ]; then
echo "Error ${http_response}: you probably need to update the list url!"
exit 1
fi
echo "(2) Processing IP addresses..."
./ipaddr2list.py "${LIST}" NDPI_PROTOCOL_CACHEFLY > "${DEST}"
rm -f "${LIST}"
echo "(3) Cachefly IPs are available in ${DEST}"
exit 0
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
cd "$(dirname "${0}")"
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_cloudflare_match.c.inc
LIST=/tmp/cloudflare.list
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
cd "$(dirname "${0}")"
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_ethereum_match.c.inc
TMP=/tmp/ethereum
+2
View File
@@ -1,5 +1,7 @@
#!/bin/sh
set -e
if [ "$#" -ne 1 ]; then
echo "Usage: $0 AS-Number" >&2
exit 1
@@ -1,5 +1,7 @@
#!/bin/sh
set -e
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_google_cloud_match.c.inc
@@ -1,5 +1,7 @@
#!/bin/sh
set -e
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_icloud_private_relay_match.c.inc
+3 -1
View File
@@ -1,6 +1,8 @@
#!/bin/sh
cd "$(dirname "${0}")"
set -e
cd "$(dirname "${0}")" || exit 1
DEST_OUTLOOK=../src/lib/inc_generated/ndpi_ms_outlook_match.c.inc
DEST_SKYPE_MSTEAMS=../src/lib/inc_generated/ndpi_ms_skype_teams_match.c.inc
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
cd "$(dirname "${0}")"
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
LIST=/tmp/tor.list
+3 -1
View File
@@ -8,6 +8,8 @@ RETVAL=0
RETVAL=$(( RETVAL + $? ))
./azure_ip_addresses_download.sh
RETVAL=$(( RETVAL + $? ))
./cachefly_ip_addresses_download.sh
RETVAL=$(( RETVAL + $? ))
./cloudflare_ip_addresses_download.sh
RETVAL=$(( RETVAL + $? ))
./ethereum_ip_addresses_download.sh
@@ -30,5 +32,5 @@ RETVAL=$(( RETVAL + $? ))
./asn_update.sh
RETVAL=$(( RETVAL + $? ))
test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} scripts failed"
test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} script(s) failed"
exit ${RETVAL}
+3 -1
View File
@@ -1,6 +1,8 @@
#!/bin/sh
cd "$(dirname "${0}")"
set -e
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_whatsapp_match.c.inc
TMP=/tmp/wa.zip
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
cd "$(dirname "${0}")"
cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_zoom_match.c.inc
LIST=/tmp/zoom.list