From b748fb1c96f179100bcff84467310b4bdf0767ee Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 16 Sep 2022 15:15:04 +0000 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D1=8F?= =?UTF-8?q?=D0=B5=D1=82=20IP-=D0=B4=D0=B8=D0=B0=D0=BF=D0=B0=D0=B7=D0=BE?= =?UTF-8?q?=D0=BD=D1=8B=20Google=20(#1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maxim Slipenko Reviewed-on: https://git.slipenko.com/Maks1mS/DPR-blocked-sites/pulls/1 --- domains.txt | 6 ++++-- google_ranges.txt | 6 ++++++ ips.txt | 21 +++++++------------ is_google_subnet.py | 19 +++++++++++++++++ runners/1_ips_from_domains_txt.py | 21 ++++++++++--------- runners/2_google_ips.py | 10 +++++---- runners/y_google_ranges.py | 34 +++++++++++++++++++++++++++++++ runners/z_remove_duplicates.py | 6 ++---- update.py | 4 +++- 9 files changed, 92 insertions(+), 35 deletions(-) create mode 100644 google_ranges.txt create mode 100644 is_google_subnet.py create mode 100644 runners/y_google_ranges.py diff --git a/domains.txt b/domains.txt index 51d6fa9..3ba0aa0 100644 --- a/domains.txt +++ b/domains.txt @@ -32,7 +32,6 @@ grafana.com # # NTP сервера Google # -time.google.com time1.google.com time2.google.com time3.google.com @@ -46,4 +45,7 @@ chess.com # # developers.google.com # -developers.google.com \ No newline at end of file +developers.google.com + +googleapis.com +firebase.google.com \ No newline at end of file diff --git a/google_ranges.txt b/google_ranges.txt new file mode 100644 index 0000000..66efb3e --- /dev/null +++ b/google_ranges.txt @@ -0,0 +1,6 @@ +34.64.0.0/10 +35.240.0.0/13 +104.196.0.0/14 +216.239.32.0/19 +173.194.0.0/16 +64.233.160.0/19 \ No newline at end of file diff --git a/ips.txt b/ips.txt index 6c35bb6..b391931 100644 --- a/ips.txt +++ b/ips.txt @@ -1,26 +1,13 @@ 192.229.221.58/32 # us.download.nvidia.com -34.120.25.175/32 # habitica.com -104.198.14.52/32 # docs.nestjs.com, nestjs.com -35.244.248.76/32 # stackshare.io 172.65.251.78/32 # gitlab.com -34.120.177.193/32 # grafana.com -216.239.35.0/32 # time.google.com, time1.google.com -216.239.35.4/32 # time2.google.com -216.239.35.8/32 # time3.google.com -216.239.35.12/32 # time4.google.com -34.117.44.137/32 # chess.com -142.251.1.101/32 # developers.google.com 35.190.247.0/24 # google (spf) -64.233.160.0/19 # google (spf) 66.102.0.0/20 # google (spf) 66.249.80.0/20 # google (spf) 72.14.192.0/18 # google (spf) 74.125.0.0/16 # google (spf) 108.177.8.0/21 # google (spf) -173.194.0.0/16 # google (spf) 209.85.128.0/17 # google (spf) 216.58.192.0/19 # google (spf) -216.239.32.0/19 # google (spf) 172.217.0.0/19 # google (spf) 172.217.32.0/20 # google (spf) 172.217.128.0/19 # google (spf) @@ -30,4 +17,10 @@ 172.253.112.0/20 # google (spf) 108.177.96.0/19 # google (spf) 35.191.0.0/16 # google (spf) -130.211.0.0/22 # google (spf) \ No newline at end of file +130.211.0.0/22 # google (spf) +34.64.0.0/10 # google_range :: habitica.com (34.120.25.175/32), grafana.com (34.120.177.193/32), chess.com (34.117.44.137/32) +104.196.0.0/14 # google_range :: docs.nestjs.com, nestjs.com (104.198.14.52/32) +35.240.0.0/13 # google_range :: stackshare.io (35.244.248.76/32) +216.239.32.0/19 # google_range :: time1.google.com (216.239.35.0/32), time2.google.com (216.239.35.4/32), time3.google.com (216.239.35.8/32), time4.google.com (216.239.35.12/32), google (spf) (216.239.32.0/19) +64.233.160.0/19 # google_range :: developers.google.com (64.233.161.138/32), firebase.google.com (64.233.164.100/32), google (spf) (64.233.160.0/19) +173.194.0.0/16 # google_range :: googleapis.com (173.194.220.105/32), google (spf) (173.194.0.0/16) \ No newline at end of file diff --git a/is_google_subnet.py b/is_google_subnet.py new file mode 100644 index 0000000..a0dad5d --- /dev/null +++ b/is_google_subnet.py @@ -0,0 +1,19 @@ +import ipaddress +import requests +import json + +x = input() + +res = requests.get('https://www.gstatic.com/ipranges/goog.json') +response = json.loads(res.text) + +ipv4Prefix = [x['ipv4Prefix'] for x in filter(lambda x: 'ipv4Prefix' in x, response["prefixes"])] + +for prefix in ipv4Prefix: + isInPrefix = ipaddress.ip_address(x) in ipaddress.ip_network(prefix) + + if isInPrefix: + print(x, 'is subnet of', prefix) + exit() + +print('not found') \ No newline at end of file diff --git a/runners/1_ips_from_domains_txt.py b/runners/1_ips_from_domains_txt.py index 6ff86e7..e7d4d58 100644 --- a/runners/1_ips_from_domains_txt.py +++ b/runners/1_ips_from_domains_txt.py @@ -5,17 +5,18 @@ from utils import prettyprint ips = [] dict = defaultdict(list) -def main(): - with open('ips.txt', 'w') as out: - for line in open('domains.txt'): - l = line.strip() +def main(file_data: list): + for line in open('domains.txt'): + l = line.strip() - if l.startswith('#') or l == '': - continue + if l.startswith('#') or l == '': + continue - dict[socket.gethostbyname(l) + '/32'].append(l) + dict[socket.gethostbyname(l) + '/32'].append(l) - ips = list(dict.keys()) + ips = list(dict.keys()) - for ip in ips: - out.write(prettyprint(ip, ', '.join(dict[ip])) + '\n') \ No newline at end of file + for ip in ips: + file_data.append(prettyprint(ip, ', '.join(dict[ip])) + '\n') + + return file_data \ No newline at end of file diff --git a/runners/2_google_ips.py b/runners/2_google_ips.py index 632b05b..876a230 100644 --- a/runners/2_google_ips.py +++ b/runners/2_google_ips.py @@ -24,9 +24,11 @@ def resolve_by_spf(domain): return list(dict.fromkeys(result)) -def main(): +def main(file_data: list): ips = resolve_by_spf("_spf.google.com") - with open('ips.txt', 'a') as out: - for ip in ips: - out.write(prettyprint(ip, 'google (spf)') + '\n') + for ip in ips: + file_data.append(prettyprint(ip, 'google (spf)') + '\n') + + return file_data + diff --git a/runners/y_google_ranges.py b/runners/y_google_ranges.py new file mode 100644 index 0000000..fb6da27 --- /dev/null +++ b/runners/y_google_ranges.py @@ -0,0 +1,34 @@ +from collections import defaultdict +from utils import prettyprint +import ipaddress + +new_ips_data = defaultdict(list) + +cloud_ranges_file = open("google_ranges.txt", "r") +cloud_ranges = [x.strip() for x in cloud_ranges_file.readlines()] + +def process_list(item: str): + l = item.strip() + + if l.startswith('#') or l == '': + return True + + [range, comment] = [x.strip() for x in l.split('#')] + + for cloud_range in cloud_ranges: + if ipaddress.ip_network(range).subnet_of(ipaddress.ip_network(cloud_range)): + new_ips_data[cloud_range].append(f'{comment} ({range})') + + return False + + return True + +def main(file_data: list[str]): + file_data = list(filter(process_list, file_data)) + + ips = list(new_ips_data.keys()) + + for ip in ips: + file_data.append(prettyprint(ip, 'google_range :: ' + ', '.join(new_ips_data[ip])) + '\n') + + return file_data \ No newline at end of file diff --git a/runners/z_remove_duplicates.py b/runners/z_remove_duplicates.py index b75cb0a..1980b40 100644 --- a/runners/z_remove_duplicates.py +++ b/runners/z_remove_duplicates.py @@ -1,7 +1,5 @@ -def main(): - inp = open('ips.txt', 'r') - uniqlines = list(dict.fromkeys(inp.readlines())) - inp.close() +def main(file_data: list[str]): + uniqlines = list(dict.fromkeys(file_data)) out = open('ips.txt', 'w') out.writelines(uniqlines) out.truncate(out.tell()-1) diff --git a/update.py b/update.py index 0a2073d..88033ac 100644 --- a/update.py +++ b/update.py @@ -10,9 +10,11 @@ scripts = glob.glob(f"./{RUNNER_FOLDER}/*.py") scripts.sort() modules = [os.path.basename(x)[:-3] for x in scripts] +file_data = [] + for m in modules: runner = importlib.import_module('.' + m, package=RUNNER_FOLDER) - runner.main() + file_data = runner.main(file_data) ''' for script in scripts: