Add support for Google Cloud (#1447)

Differentiate between Google its own apps/services and Google Cloud.
We already do something similar for Amazon vs AWS and Microsoft vs Azure.
This commit is contained in:
Ivan Nardi
2022-02-20 10:19:52 +01:00
committed by GitHub
parent 75a53ad064
commit 2f5f445f72
18 changed files with 797 additions and 99 deletions
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
import json
import urllib.request
import netaddr
GOOG_URL="https://www.gstatic.com/ipranges/goog.json"
CLOUD_URL="https://www.gstatic.com/ipranges/cloud.json"
def read_url(url):
try:
s = urllib.request.urlopen(url).read()
return json.loads(s)
except urllib.error.HTTPError:
print("Invalid HTTP response from %s" % url)
return {}
except json.decoder.JSONDecodeError:
print("Could not parse HTTP response from %s" % url)
return {}
def main():
goog_json=read_url(GOOG_URL)
cloud_json=read_url(CLOUD_URL)
if goog_json and cloud_json:
# print("{} published: {}".format(GOOG_URL,goog_json.get('creationTime')))
# print("{} published: {}".format(CLOUD_URL,cloud_json.get('creationTime')))
goog_cidrs = netaddr.IPSet()
for pref in goog_json['prefixes']:
if pref.get('ipv4Prefix'):
goog_cidrs.add(pref.get('ipv4Prefix'))
cloud_cidrs = netaddr.IPSet()
for pref in cloud_json['prefixes']:
if pref.get('ipv4Prefix'):
cloud_cidrs.add(pref.get('ipv4Prefix'))
# print("IP ranges for Google APIs and services default domains:")
for i in goog_cidrs.difference(cloud_cidrs).iter_cidrs():
print(i)
if __name__=='__main__':
main()
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
cd "$(dirname "${0}")" || return
DEST=../src/lib/ndpi_google_cloud_match.c.inc
TMP=/tmp/google_c.json
LIST=/tmp/google_c.list
ORIGIN="https://www.gstatic.com/ipranges/cloud.json"
echo "(1) Downloading file..."
http_response=$(curl -s -o $TMP -w "%{http_code}" ${ORIGIN})
if [ "$http_response" != "200" ]; then
echo "Error $http_response: you probably need to update the list url!"
return
fi
echo "(2) Processing IP addresses..."
jq -r '.prefixes | .[].ipv4Prefix | select( . != null )' $TMP > $LIST # TODO: ipv6
./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE_CLOUD > $DEST
rm -f $TMP $LIST
echo "(3) Google Cloud IPs are available in $DEST"
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
cd "$(dirname "${0}")" || return
DEST=../src/lib/ndpi_google_match.c.inc
LIST=/tmp/google.list
echo "(1) Downloading file..."
#Nothing to do
echo "(2) Processing IP addresses..."
#https://cloud.google.com/vpc/docs/configure-private-google-access#ip-addr-defaults
python3 google.py > $LIST
./ipaddr2list.py $LIST NDPI_PROTOCOL_GOOGLE > $DEST
#rm -f $TMP $LIST
echo "(3) Google IPs are available in $DEST"
@@ -10,3 +10,5 @@ cd "$(dirname "${0}")"
./tor_ip_addresses_download.sh
./whatsapp_ip_addresses_download.sh
./zoom_ip_addresses_download.sh
./google_cloud_ip_addresses_download.sh
./google_ip_addresses_download.sh