Add few scripts to easily update some IPs lists (#1436)
* Add few scripts to easily update some IPs lists Some IPs lists should be updated frequently: try to easy the process. The basic idea is taken fromd59fefd0anda8fe74e5(for Azure addresses): one specific .c.inc file and one script for each protocol. Add the possibility to don't load a specific list. Rename the old NDPI_PROTOCOL_HOTMAIL id to NDPI_PROTOCOL_MS_OUTLOOK, to identify Hotmail/Outlook/Exchange flows. TODO: ipv6 Remove the 9 addresses associated to BitTorrent: they have been added ine2f21116but it is not clear why all the traffic to/from these ips should be classified as BitTorrent. * Added quotes * Added quotes Co-authored-by: Luca Deri <lucaderi@users.noreply.github.com>
This commit is contained in:
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_amazon_aws_match.c.inc
|
||||
TMP=/tmp/aws.json
|
||||
LIST=/tmp/aws.list
|
||||
ORIGIN=https://ip-ranges.amazonaws.com/ip-ranges.json
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
jq -r '.prefixes | .[].ip_prefix' $TMP > $LIST # TODO: ipv6
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_AMAZON_AWS > $DEST
|
||||
rm -f $TMP $LIST
|
||||
|
||||
echo "(3) Amazon AWS IPs are available in $DEST"
|
||||
@@ -1,19 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
OUT=../src/lib/ndpi_azure_match.c.inc
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_azure_match.c.inc
|
||||
TMP=/tmp/azure.json
|
||||
LIST=/tmp/azure.list
|
||||
# https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
|
||||
ORIGIN="https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20220124.json"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
# https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
|
||||
curl -s https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20211213.json -o $TMP
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
# Note: the last "grep -v :" is used to skip IPv6 addresses
|
||||
tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $OUT
|
||||
./ipaddr2list.py $OUT NDPI_PROTOCOL_MICROSOFT_AZURE > $TMP
|
||||
/bin/mv $TMP $OUT
|
||||
tr -d '\r' < $TMP | grep / | tr -d '"' | tr -d " " | tr -d "," | grep -v : > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_AZURE > $DEST
|
||||
rm -f $TMP $LIST
|
||||
|
||||
echo "(3) Microsoft Azure IPs are available in $OUT"
|
||||
echo "(3) Microsoft Azure IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_cloudflare_match.c.inc
|
||||
LIST=/tmp/cloudflare.list
|
||||
# TODO: ipv6 list from https://www.cloudflare.com/ips-v6
|
||||
ORIGIN="https://www.cloudflare.com/ips-v4"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_CLOUDFLARE > $DEST
|
||||
rm -f $LIST
|
||||
|
||||
echo "(3) Cloudflare IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_ethereum_match.c.inc
|
||||
TMP=/tmp/ethereum
|
||||
LIST=/tmp/ethereum.list
|
||||
ORIGIN="https://raw.githubusercontent.com/ethereum/go-ethereum/master/params/bootnodes.go"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
grep 'enode' $TMP | grep -v '^/' | grep ':' | cut -d '@' -f 2 | cut -d ':' -f 1 > $LIST
|
||||
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MINING > $DEST
|
||||
rm -f $TMP $LIST
|
||||
|
||||
echo "(3) Ethereum/Mining IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST_OUTLOOK=../src/lib/ndpi_ms_outlook_match.c.inc
|
||||
DEST_SKYPE_MSTEAMS=../src/lib/ndpi_ms_skype_teams_match.c.inc
|
||||
DEST_ONEDRIVE=../src/lib/ndpi_ms_onedrive_match.c.inc
|
||||
DEST_OFFICE365=../src/lib/ndpi_ms_office365_match.c.inc
|
||||
TMP=/tmp/ms.json
|
||||
LIST=/tmp/ms.list
|
||||
# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
|
||||
ORIGIN="https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed1-bad1-445f-b386-b919946339a7"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
|
||||
#OUTLOOK
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Exchange") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_OUTLOOK > $DEST_OUTLOOK
|
||||
|
||||
#SKYPE/TEAMS
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Skype") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_SKYPE_TEAMS > $DEST_SKYPE_MSTEAMS
|
||||
|
||||
#ONEDRIVE
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="SharePoint") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MS_ONE_DRIVE > $DEST_ONEDRIVE
|
||||
|
||||
#OFFICE
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
jq -r '.[] | select(.serviceArea=="Common") | .ips[]?' < $TMP | grep -v ':' | sort -u | uniq > $LIST
|
||||
#TODO: NDPI_PROTOCOL_MICROSOFT_365 or NDPI_PROTOCOL_MICROSOFT?
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MICROSOFT_365 > $DEST_OFFICE365
|
||||
|
||||
rm -f $TMP $LIST
|
||||
|
||||
echo "(3) Microsoft IPs are available in $DEST_OUTLOOK, $DEST_SKYPE_MSTEAMS, $DEST_ONEDRIVE, $DEST_OFFICE365"
|
||||
|
||||
|
||||
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_tor_match.c.inc
|
||||
LIST=/tmp/tor.list
|
||||
# There are at least two lists:
|
||||
# * https://torstatus.rueckgr.at/ip_list_all.php/Tor_ip_list_ALL.csv
|
||||
# * https://check.torproject.org/torbulkexitlist
|
||||
# The latter seems to be more "stable" (the former changes every few seconds!)
|
||||
ORIGIN="https://check.torproject.org/torbulkexitlist"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ $http_response != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST
|
||||
rm -f $LIST
|
||||
|
||||
echo "(3) TOR IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
./aws_ip_addresses_download.sh
|
||||
./azure_ip_addresses_download.sh
|
||||
./cloudflare_ip_addresses_download.sh
|
||||
./ethereum_ip_addresses_download.sh
|
||||
./microsoft_ip_addresses_download.sh
|
||||
./tor_ip_addresses_download.sh
|
||||
./whatsapp_ip_addresses_download.sh
|
||||
./zoom_ip_addresses_download.sh
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_whatsapp_match.c.inc
|
||||
TMP=/tmp/wa.zip
|
||||
LIST=/tmp/wa.list
|
||||
# https://developers.facebook.com/docs/whatsapp/guides/network-requirements/
|
||||
ORIGIN="https://scontent.fmxp6-1.fna.fbcdn.net/v/t39.8562-6/218944277_794653217800107_785885630662402277_n.zip?_nc_cat=102&ccb=1-5&_nc_sid=ae5e01&_nc_ohc=CxWH4uR6uPsAX-Yga3M&_nc_ht=scontent.fmxp6-1.fna&oh=00_AT9gC0NiHKwmgoBdNX9jbVbxtciJ8HzeGdOLj35n3kWeUw&oe=6201B6A9"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
zcat $TMP > $LIST
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_WHATSAPP > $DEST
|
||||
rm -f $TMP $LIST
|
||||
|
||||
echo "(3) WhatsApp IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
DEST=../src/lib/ndpi_zoom_match.c.inc
|
||||
LIST=/tmp/zoom.list
|
||||
# https://support.zoom.us/hc/en-us/articles/201362683-Zoom-network-firewall-or-proxy-server-settings
|
||||
# There are few lists in this page, partially overlapping. Pick the generic one
|
||||
ORIGIN="https://assets.zoom.us/docs/ipranges/Zoom.txt"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_ZOOM > $DEST
|
||||
rm -f $LIST
|
||||
|
||||
echo "(3) ZOOM IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user