Ip address list: aggregate Mullvad and Tor lists too (#2154)

Missing from bdb73db1a
See #2150
This commit is contained in:
Ivan Nardi
2023-11-21 08:28:32 +01:00
committed by GitHub
parent 8d71998670
commit 55664392a9
5 changed files with 1957 additions and 2489 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1
View File
@@ -14,6 +14,7 @@ if len(sys.argv) < 2:
ipFile = open(sys.argv[1])
ipAddresses = list(ipFile.readlines())
ipAddresses = [x.replace("\n","") for x in ipAddresses]
ipAddresses = sorted(ipAddresses)
cidrs = netaddr.cidr_merge(ipAddresses)
+8 -2
View File
@@ -9,6 +9,8 @@ DEST=../src/lib/inc_generated/ndpi_mullvad_match.c.inc
TMP=/tmp/mullvad.json
LIST=/tmp/mullvad.list
LIST6=/tmp/mullvad.list6
LIST_MERGED=/tmp/mullvad.list_m
LIST6_MERGED=/tmp/mullvad.list6_m
ORIGIN=https://api-www.mullvad.net/www/relays/all/
@@ -20,12 +22,16 @@ is_file_empty "${TMP}"
echo "(2) Processing IP addresses..."
jq -r '.[].ipv4_addr_in' $TMP > $LIST
is_file_empty "${LIST}"
./mergeipaddrlist.py $LIST > $LIST_MERGED
is_file_empty "${LIST_MERGED}"
jq -r '.[].ipv6_addr_in | select( . != null )' $TMP > $LIST6
is_file_empty "${LIST6}"
./ipaddr2list.py $LIST NDPI_PROTOCOL_MULLVAD $LIST6 > $DEST
./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
is_file_empty "${LIST6_MERGED}"
./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MULLVAD $LIST6_MERGED > $DEST
is_file_empty "${DEST}"
rm -f $TMP $LIST $LIST6
rm -f $TMP $LIST $LIST6 $LIST_MERGED $LIST6_MERGED
echo "(3) Mullvad IPs are available in $DEST"
exit 0
+5 -2
View File
@@ -7,6 +7,7 @@ cd "$(dirname "${0}")" || exit 1
DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
LIST=/tmp/tor.list
LIST_MERGED=/tmp/tor.list_m
# There are at least two lists:
# * https://torstatus.rueckgr.at/ip_list_all.php/Tor_ip_list_ALL.csv
# * https://check.torproject.org/torbulkexitlist
@@ -18,12 +19,14 @@ echo "(1) Downloading file... ${ORIGIN}"
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
check_http_response "${http_response}"
is_file_empty "${LIST}"
./mergeipaddrlist.py $LIST > $LIST_MERGED
is_file_empty "${LIST_MERGED}"
#TODO: TOR relays don't support ipv6 yet
echo "(2) Processing IP addresses..."
./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST
rm -f "${LIST}"
./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TOR > $DEST
rm -f "${LIST}" "${LIST_MERGED}"
is_file_empty "${DEST}"
echo "(3) TOR IPs are available in $DEST"