Добавляет IP-диапазоны Google (#1)

Co-authored-by: Maxim Slipenko <no-reply@maxim.slipenko.com>
Reviewed-on: #1

domains.txt

@@ -32,7 +32,6 @@ grafana.com
 #
 # NTP сервера Google
 #
-time.google.com
 time1.google.com
 time2.google.com
 time3.google.com
@@ -46,4 +45,7 @@ chess.com
 #
 # developers.google.com
 #
-developers.google.com
+developers.google.com
+
+googleapis.com
+firebase.google.com

google_ranges.txt

@@ -0,0 +1,6 @@
+34.64.0.0/10
+35.240.0.0/13
+104.196.0.0/14
+216.239.32.0/19
+173.194.0.0/16
+64.233.160.0/19

ips.txt

@@ -1,26 +1,13 @@
 192.229.221.58/32   # us.download.nvidia.com
-34.120.25.175/32    # habitica.com
-104.198.14.52/32    # docs.nestjs.com, nestjs.com
-35.244.248.76/32    # stackshare.io
 172.65.251.78/32    # gitlab.com
-34.120.177.193/32   # grafana.com
-216.239.35.0/32     # time.google.com, time1.google.com
-216.239.35.4/32     # time2.google.com
-216.239.35.8/32     # time3.google.com
-216.239.35.12/32    # time4.google.com
-34.117.44.137/32    # chess.com
-142.251.1.101/32    # developers.google.com
 35.190.247.0/24     # google (spf)
-64.233.160.0/19     # google (spf)
 66.102.0.0/20       # google (spf)
 66.249.80.0/20      # google (spf)
 72.14.192.0/18      # google (spf)
 74.125.0.0/16       # google (spf)
 108.177.8.0/21      # google (spf)
-173.194.0.0/16      # google (spf)
 209.85.128.0/17     # google (spf)
 216.58.192.0/19     # google (spf)
-216.239.32.0/19     # google (spf)
 172.217.0.0/19      # google (spf)
 172.217.32.0/20     # google (spf)
 172.217.128.0/19    # google (spf)
@@ -30,4 +17,10 @@
 172.253.112.0/20    # google (spf)
 108.177.96.0/19     # google (spf)
 35.191.0.0/16       # google (spf)
-130.211.0.0/22      # google (spf)
+130.211.0.0/22      # google (spf)
+34.64.0.0/10        # google_range :: habitica.com (34.120.25.175/32), grafana.com (34.120.177.193/32), chess.com (34.117.44.137/32)
+104.196.0.0/14      # google_range :: docs.nestjs.com, nestjs.com (104.198.14.52/32)
+35.240.0.0/13       # google_range :: stackshare.io (35.244.248.76/32)
+216.239.32.0/19     # google_range :: time1.google.com (216.239.35.0/32), time2.google.com (216.239.35.4/32), time3.google.com (216.239.35.8/32), time4.google.com (216.239.35.12/32), google (spf) (216.239.32.0/19)
+64.233.160.0/19     # google_range :: developers.google.com (64.233.161.138/32), firebase.google.com (64.233.164.100/32), google (spf) (64.233.160.0/19)
+173.194.0.0/16      # google_range :: googleapis.com (173.194.220.105/32), google (spf) (173.194.0.0/16)

is_google_subnet.py

@@ -0,0 +1,19 @@
+import ipaddress
+import requests
+import json 
+
+x = input()
+
+res = requests.get('https://www.gstatic.com/ipranges/goog.json')
+response = json.loads(res.text)
+
+ipv4Prefix = [x['ipv4Prefix'] for x in filter(lambda x: 'ipv4Prefix' in x, response["prefixes"])]
+
+for prefix in ipv4Prefix:
+    isInPrefix = ipaddress.ip_address(x) in ipaddress.ip_network(prefix)
+
+    if isInPrefix:
+        print(x, 'is subnet of', prefix)
+        exit()
+
+print('not found')

runners/1_ips_from_domains_txt.py

@@ -5,17 +5,18 @@ from utils import prettyprint
 ips = []
 dict = defaultdict(list)
 
-def main():
-    with open('ips.txt', 'w') as out:
-        for line in open('domains.txt'):
-            l = line.strip()
+def main(file_data: list):
+    for line in open('domains.txt'):
+        l = line.strip()
 
-            if l.startswith('#') or l == '':
-                continue
+        if l.startswith('#') or l == '':
+            continue
 
-            dict[socket.gethostbyname(l) + '/32'].append(l)
+        dict[socket.gethostbyname(l) + '/32'].append(l)
 
-        ips = list(dict.keys())
+    ips = list(dict.keys())
 
-        for ip in ips:
-            out.write(prettyprint(ip, ', '.join(dict[ip])) + '\n')
+    for ip in ips:
+        file_data.append(prettyprint(ip, ', '.join(dict[ip])) + '\n')
+
+    return file_data

runners/2_google_ips.py

@@ -24,9 +24,11 @@ def resolve_by_spf(domain):
 
         return list(dict.fromkeys(result))
 
-def main():
+def main(file_data: list):
     ips = resolve_by_spf("_spf.google.com")
 
-    with open('ips.txt', 'a') as out:
-        for ip in ips:
-            out.write(prettyprint(ip, 'google (spf)') + '\n')
+    for ip in ips:
+        file_data.append(prettyprint(ip, 'google (spf)') + '\n')
+
+    return file_data
+

runners/y_google_ranges.py

@@ -0,0 +1,34 @@
+from collections import defaultdict
+from utils import prettyprint
+import ipaddress
+
+new_ips_data = defaultdict(list)
+
+cloud_ranges_file = open("google_ranges.txt", "r")
+cloud_ranges = [x.strip() for x in cloud_ranges_file.readlines()]
+
+def process_list(item: str):
+    l = item.strip()
+
+    if l.startswith('#') or l == '':
+        return True
+
+    [range, comment] = [x.strip() for x in l.split('#')]
+
+    for cloud_range in cloud_ranges:
+        if ipaddress.ip_network(range).subnet_of(ipaddress.ip_network(cloud_range)):
+            new_ips_data[cloud_range].append(f'{comment} ({range})')
+
+            return False
+
+    return True
+
+def main(file_data: list[str]):
+    file_data = list(filter(process_list, file_data))
+
+    ips = list(new_ips_data.keys())
+
+    for ip in ips:        
+        file_data.append(prettyprint(ip, 'google_range :: ' + ', '.join(new_ips_data[ip])) + '\n')
+
+    return file_data

runners/z_remove_duplicates.py

@@ -1,7 +1,5 @@
-def main():
-    inp = open('ips.txt', 'r')
-    uniqlines = list(dict.fromkeys(inp.readlines()))
-    inp.close()
+def main(file_data: list[str]):
+    uniqlines = list(dict.fromkeys(file_data))
     out = open('ips.txt', 'w')
     out.writelines(uniqlines)
     out.truncate(out.tell()-1)

update.py

@@ -10,9 +10,11 @@ scripts = glob.glob(f"./{RUNNER_FOLDER}/*.py")
 scripts.sort()
 modules = [os.path.basename(x)[:-3] for x in scripts]
 
+file_data = []
+
 for m in modules:
     runner = importlib.import_module('.' + m, package=RUNNER_FOLDER)
-    runner.main()
+    file_data = runner.main(file_data)
 
 '''
 for script in scripts: