Ip address list: aggregate Mullvad and Tor lists too (#2154)
Missing from bdb73db1a
See #2150
This commit is contained in:
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,7 @@ if len(sys.argv) < 2:
|
||||
|
||||
ipFile = open(sys.argv[1])
|
||||
ipAddresses = list(ipFile.readlines())
|
||||
ipAddresses = [x.replace("\n","") for x in ipAddresses]
|
||||
ipAddresses = sorted(ipAddresses)
|
||||
cidrs = netaddr.cidr_merge(ipAddresses)
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ DEST=../src/lib/inc_generated/ndpi_mullvad_match.c.inc
|
||||
TMP=/tmp/mullvad.json
|
||||
LIST=/tmp/mullvad.list
|
||||
LIST6=/tmp/mullvad.list6
|
||||
LIST_MERGED=/tmp/mullvad.list_m
|
||||
LIST6_MERGED=/tmp/mullvad.list6_m
|
||||
ORIGIN=https://api-www.mullvad.net/www/relays/all/
|
||||
|
||||
|
||||
@@ -20,12 +22,16 @@ is_file_empty "${TMP}"
|
||||
echo "(2) Processing IP addresses..."
|
||||
jq -r '.[].ipv4_addr_in' $TMP > $LIST
|
||||
is_file_empty "${LIST}"
|
||||
./mergeipaddrlist.py $LIST > $LIST_MERGED
|
||||
is_file_empty "${LIST_MERGED}"
|
||||
jq -r '.[].ipv6_addr_in | select( . != null )' $TMP > $LIST6
|
||||
is_file_empty "${LIST6}"
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_MULLVAD $LIST6 > $DEST
|
||||
./mergeipaddrlist.py $LIST6 > $LIST6_MERGED
|
||||
is_file_empty "${LIST6_MERGED}"
|
||||
./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_MULLVAD $LIST6_MERGED > $DEST
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
rm -f $TMP $LIST $LIST6
|
||||
rm -f $TMP $LIST $LIST6 $LIST_MERGED $LIST6_MERGED
|
||||
|
||||
echo "(3) Mullvad IPs are available in $DEST"
|
||||
exit 0
|
||||
|
||||
@@ -7,6 +7,7 @@ cd "$(dirname "${0}")" || exit 1
|
||||
|
||||
DEST=../src/lib/inc_generated/ndpi_tor_match.c.inc
|
||||
LIST=/tmp/tor.list
|
||||
LIST_MERGED=/tmp/tor.list_m
|
||||
# There are at least two lists:
|
||||
# * https://torstatus.rueckgr.at/ip_list_all.php/Tor_ip_list_ALL.csv
|
||||
# * https://check.torproject.org/torbulkexitlist
|
||||
@@ -18,12 +19,14 @@ echo "(1) Downloading file... ${ORIGIN}"
|
||||
http_response=$(curl -s -o $LIST -w "%{http_code}" ${ORIGIN})
|
||||
check_http_response "${http_response}"
|
||||
is_file_empty "${LIST}"
|
||||
./mergeipaddrlist.py $LIST > $LIST_MERGED
|
||||
is_file_empty "${LIST_MERGED}"
|
||||
|
||||
#TODO: TOR relays don't support ipv6 yet
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
./ipaddr2list.py $LIST NDPI_PROTOCOL_TOR > $DEST
|
||||
rm -f "${LIST}"
|
||||
./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_TOR > $DEST
|
||||
rm -f "${LIST}" "${LIST_MERGED}"
|
||||
is_file_empty "${DEST}"
|
||||
|
||||
echo "(3) TOR IPs are available in $DEST"
|
||||
|
||||
Reference in New Issue
Block a user