From 0f7c5bea2fee3159e196b9aebc127be55f599523 Mon Sep 17 00:00:00 2001 From: darkclip Date: Wed, 19 Jul 2023 21:50:46 +0800 Subject: [PATCH] update --- .github/workflows/build-net.yml | 50 +++++++++++ .gitignore | 2 + exp_net.sh | 19 +++++ exp_ros.sh | 19 +++++ netgen.py | 145 ++++++++++++++++++++++++++++++++ private.txt | 8 ++ private6.txt | 4 + ray.proto | 102 ++++++++++++++++++++++ ray_pb2.py | 50 +++++++++++ raydata.py | 136 ++++++++++++++++++++++++++++++ requirements.txt | 2 + 11 files changed, 537 insertions(+) create mode 100644 .github/workflows/build-net.yml create mode 100644 .gitignore create mode 100755 exp_net.sh create mode 100755 exp_ros.sh create mode 100755 netgen.py create mode 100644 private.txt create mode 100644 private6.txt create mode 100644 ray.proto create mode 100644 ray_pb2.py create mode 100755 raydata.py create mode 100644 requirements.txt diff --git a/.github/workflows/build-net.yml b/.github/workflows/build-net.yml new file mode 100644 index 00000000..e2e6c31f --- /dev/null +++ b/.github/workflows/build-net.yml @@ -0,0 +1,50 @@ +name: build net + +on: + push: + branches: ['*'] + schedule: + - cron: '0 16 * * *' + +jobs: + build-net: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Check out ip-lists branch of gaoyifan/china-operator-ip + uses: actions/checkout@v3 + with: + repository: gaoyifan/china-operator-ip + ref: ip-lists + path: ip-lists + - name: Checkout net branch + uses: actions/checkout@v3 + with: + ref: net + path: net + - name: Rebuild net branch + if: github.event_name == 'schedule' + run: | + rm net/* + ./exp_net.sh + - name: Push to net branch + if: github.event_name == 'schedule' + continue-on-error: true + run: | + cd net + tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git add . + git commit -m "update $(date +%Y-%m-%d)" + git push -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3af6a7b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +/venv \ No newline at end of file diff --git a/exp_net.sh b/exp_net.sh new file mode 100755 index 00000000..cadea419 --- /dev/null +++ b/exp_net.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +./netgen.py ip-lists/china.txt -o net/china.txt +./netgen.py ip-lists/china6.txt -o net/china6.txt + +./netgen.py ip-lists/chinanet.txt -o net/telecom.txt +./netgen.py ip-lists/chinanet6.txt -o net/telecom6.txt + +./netgen.py ip-lists/unicom.txt -o net/unicom.txt +./netgen.py ip-lists/unicom6.txt -o net/unicom6.txt + +./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o net/others.txt +./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o net/others6.txt + +./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o net/foreign.txt +./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o net/foreign6.txt + +./netgen.py private.txt -o net/private.txt +./netgen.py private6.txt -o net/private6.txt diff --git a/exp_ros.sh b/exp_ros.sh new file mode 100755 index 00000000..2b6e7cac --- /dev/null +++ b/exp_ros.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +./netgen.py ip-lists/china.txt -o ros/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' +./netgen.py ip-lists/china6.txt -o ros/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' + +./netgen.py ip-lists/chinanet.txt -o ros/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' +./netgen.py ip-lists/chinanet6.txt -o ros/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' + +./netgen.py ip-lists/unicom.txt -o ros/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' +./netgen.py ip-lists/unicom6.txt -o ros/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' + +./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o ros/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' +./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o ros/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' + +./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o ros/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' +./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o ros/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' + +./netgen.py private.txt -o ros/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' +./netgen.py private6.txt -o ros/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' diff --git a/netgen.py b/netgen.py new file mode 100755 index 00000000..e7c06397 --- /dev/null +++ b/netgen.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import typing +import sys +import os +from pathlib import Path +from ipaddress import IPv4Network, IPv6Network, AddressValueError + +try: + from aggregate_prefixes import aggregate_prefixes +except ImportError: + aggregate_prefixes = None + + +def prepare_networks( + nets_or_files: typing.List[str], only4: bool = False, only6: bool = False +) -> typing.List[IPv4Network | IPv6Network]: + networks = [] + for net in nets_or_files: + if not only6: + try: + networks.append(IPv4Network(net)) + except AddressValueError: + pass + if not only4: + try: + networks.append(IPv6Network(net)) + except AddressValueError: + pass + for file in nets_or_files: + file_path = Path(file).resolve() + if not file_path.exists() or not file_path.is_file(): + continue + with file_path.open(encoding='utf-8') as fp: + for line in fp: + if not only6: + try: + networks.append(IPv4Network(line.strip())) + except AddressValueError: + pass + if not only4: + try: + networks.append(IPv6Network(line.strip())) + except AddressValueError: + pass + return networks + + +def find_updated_networks( + networks: typing.List[IPv4Network | IPv6Network], subnet: IPv4Network | IPv6Network +) -> typing.List[IPv4Network | IPv6Network]: + copied_networks = networks.copy() + for index, network in enumerate(networks): + try: + if subnet.subnet_of(network): + copied_networks.pop(index) + copied_networks.extend(list(network.address_exclude(subnet))) + except TypeError: + pass + return copied_networks + + +def main(args): + includes = getattr(args, 'include', []) + excludes = getattr(args, 'exclude', []) + only4 = getattr(args, 'ipv4', False) + only6 = getattr(args, 'ipv6', False) + output = getattr(args, 'output', None) + + in_networks = prepare_networks(includes) + if len(in_networks) == 0: + print('No legal include ip networks.') + sys.exit(1) + ex_networks = prepare_networks(excludes) + out_networks = in_networks.copy() + for exclude in ex_networks: + out_networks = find_updated_networks(out_networks, exclude) + if aggregate_prefixes is not None: + out4 = [] + out6 = [] + for net in out_networks: + if not only6 and isinstance(net, IPv4Network): + out4.append(net) + if not only4 and isinstance(net, IPv6Network): + out6.append(net) + out4 = aggregate_prefixes(out4) + out6 = aggregate_prefixes(out6) + out_networks = [] + out_networks.extend(out4) + out_networks.extend(out6) + + output_lines = [ + f"{getattr(args,'prefix','')}{str(network)}{getattr(args,'suffix','')}" + for network in out_networks + ] + + if output is None: + for line in output_lines: + print(line, end='') + else: + print(f'Included Networks: {len(in_networks)}') + output: Path + output_file = output.resolve() + if not output_file.parent.exists(): + os.makedirs(output_file.parent) + with output_file.open('w', encoding='utf-8') as wfp: + wfp.writelines(output_lines) + print(f'Generated Networks: {len(output_lines)}') + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + 'include', + nargs='+', + type=str, + default=[], + help='include networks (subnets or files separated by space)', + ) + parser.add_argument( + '-e', + '--exclude', + nargs='+', + type=str, + default=[], + help='exclude networks (subnets or files separated by space)', + ) + parser.add_argument('-4', '--ipv4', action='store_true', help='process ipv4 only') + parser.add_argument('-6', '--ipv6', action='store_true', help='process ipv6 only') + parser.add_argument('-o', '--output', type=Path, help='output file to write') + parser.add_argument( + '-p', '--prefix', type=str, default='', help='write line prefix' + ) + parser.add_argument( + '-s', '--suffix', type=str, default='\n', help='write line suffix' + ) + parser.set_defaults(func=main) + args = parser.parse_args() + args.func(args) diff --git a/private.txt b/private.txt new file mode 100644 index 00000000..01e6095f --- /dev/null +++ b/private.txt @@ -0,0 +1,8 @@ +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +127.0.0.0/8 +169.254.0.0/16 +198.18.0.0/15 +224.0.0.0/4 +240.0.0.0/4 \ No newline at end of file diff --git a/private6.txt b/private6.txt new file mode 100644 index 00000000..21478581 --- /dev/null +++ b/private6.txt @@ -0,0 +1,4 @@ +::1/128 +fc00::/7 +fe80::/10 +ff00::/8 \ No newline at end of file diff --git a/ray.proto b/ray.proto new file mode 100644 index 00000000..71bffa51 --- /dev/null +++ b/ray.proto @@ -0,0 +1,102 @@ +// source: app/router/routercommon/common.proto +// source: common/protoext/extensions.proto +// generate: protoc --python_out=./ ray.proto + +syntax = "proto3"; +package routercommon; +import "google/protobuf/descriptor.proto"; + +message FieldOpt { + repeated string any_wants = 1; + repeated string allowed_values = 2; + repeated string allowed_value_types = 3; + + // convert_time_read_file_into read a file into another field, and clear this + // field during input parsing + string convert_time_read_file_into = 4; + // forbidden marks a boolean to be inaccessible to user + bool forbidden = 5; + // convert_time_resource_loading read a file, and place its resource hash into + // another field + string convert_time_resource_loading = 6; + // convert_time_parse_ip parse a string ip address, and put its binary + // representation into another field + string convert_time_parse_ip = 7; +} + +extend google.protobuf.FieldOptions { FieldOpt field_opt = 50000; } + +// Domain for routing decision. +message Domain { + // Type of domain value. + enum Type { + // The value is used as is. + Plain = 0; + // The value is used as a regular expression. + Regex = 1; + // The value is a root domain. + RootDomain = 2; + // The value is a domain. + Full = 3; + } + + // Domain matching type. + Type type = 1; + + // Domain value. + string value = 2; + + message Attribute { + string key = 1; + + oneof typed_value { + bool bool_value = 2; + int64 int_value = 3; + } + } + + // Attributes of this domain. May be used for filtering. + repeated Attribute attribute = 3; +} + +// IP for routing decision, in CIDR form. +message CIDR { + // IP address, should be either 4 or 16 bytes. + bytes ip = 1; + + // Number of leading ones in the network mask. + uint32 prefix = 2; + + string ip_addr = 68000 [ (field_opt).convert_time_parse_ip = "ip" ]; +} + +message GeoIP { + string country_code = 1; + repeated CIDR cidr = 2; + bool inverse_match = 3; + + // resource_hash instruct simplified config converter to load domain from geo + // file. + bytes resource_hash = 4; + string code = 5; + + string file_path = 68000 + [ (field_opt).convert_time_resource_loading = "resource_hash" ]; +} + +message GeoIPList { repeated GeoIP entry = 1; } + +message GeoSite { + string country_code = 1; + repeated Domain domain = 2; + + // resource_hash instruct simplified config converter to load domain from geo + // file. + bytes resource_hash = 3; + string code = 4; + + string file_path = 68000 + [ (field_opt).convert_time_resource_loading = "resource_hash" ]; +} + +message GeoSiteList { repeated GeoSite entry = 1; } diff --git a/ray_pb2.py b/ray_pb2.py new file mode 100644 index 00000000..1eb819f5 --- /dev/null +++ b/ray_pb2.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ray.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tray.proto\x12\x0croutercommon\x1a google/protobuf/descriptor.proto\"\xd0\x01\n\x08\x46ieldOpt\x12\x11\n\tany_wants\x18\x01 \x03(\t\x12\x16\n\x0e\x61llowed_values\x18\x02 \x03(\t\x12\x1b\n\x13\x61llowed_value_types\x18\x03 \x03(\t\x12#\n\x1b\x63onvert_time_read_file_into\x18\x04 \x01(\t\x12\x11\n\tforbidden\x18\x05 \x01(\x08\x12%\n\x1d\x63onvert_time_resource_loading\x18\x06 \x01(\t\x12\x1d\n\x15\x63onvert_time_parse_ip\x18\x07 \x01(\t\"\xff\x01\n\x06\x44omain\x12\'\n\x04type\x18\x01 \x01(\x0e\x32\x19.routercommon.Domain.Type\x12\r\n\x05value\x18\x02 \x01(\t\x12\x31\n\tattribute\x18\x03 \x03(\x0b\x32\x1e.routercommon.Domain.Attribute\x1aR\n\tAttribute\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x42\r\n\x0btyped_value\"6\n\x04Type\x12\t\n\x05Plain\x10\x00\x12\t\n\x05Regex\x10\x01\x12\x0e\n\nRootDomain\x10\x02\x12\x08\n\x04\x46ull\x10\x03\"?\n\x04\x43IDR\x12\n\n\x02ip\x18\x01 \x01(\x0c\x12\x0e\n\x06prefix\x18\x02 \x01(\r\x12\x1b\n\x07ip_addr\x18\xa0\x93\x04 \x01(\tB\x08\x82\xb5\x18\x04:\x02ip\"\xa5\x01\n\x05GeoIP\x12\x14\n\x0c\x63ountry_code\x18\x01 \x01(\t\x12 \n\x04\x63idr\x18\x02 \x03(\x0b\x32\x12.routercommon.CIDR\x12\x15\n\rinverse_match\x18\x03 \x01(\x08\x12\x15\n\rresource_hash\x18\x04 \x01(\x0c\x12\x0c\n\x04\x63ode\x18\x05 \x01(\t\x12(\n\tfile_path\x18\xa0\x93\x04 \x01(\tB\x13\x82\xb5\x18\x0f\x32\rresource_hash\"/\n\tGeoIPList\x12\"\n\x05\x65ntry\x18\x01 \x03(\x0b\x32\x13.routercommon.GeoIP\"\x94\x01\n\x07GeoSite\x12\x14\n\x0c\x63ountry_code\x18\x01 \x01(\t\x12$\n\x06\x64omain\x18\x02 \x03(\x0b\x32\x14.routercommon.Domain\x12\x15\n\rresource_hash\x18\x03 \x01(\x0c\x12\x0c\n\x04\x63ode\x18\x04 \x01(\t\x12(\n\tfile_path\x18\xa0\x93\x04 \x01(\tB\x13\x82\xb5\x18\x0f\x32\rresource_hash\"3\n\x0bGeoSiteList\x12$\n\x05\x65ntry\x18\x01 \x03(\x0b\x32\x15.routercommon.GeoSite:J\n\tfield_opt\x12\x1d.google.protobuf.FieldOptions\x18\xd0\x86\x03 \x01(\x0b\x32\x16.routercommon.FieldOptb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ray_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(field_opt) + + DESCRIPTOR._options = None + _CIDR.fields_by_name['ip_addr']._options = None + _CIDR.fields_by_name['ip_addr']._serialized_options = b'\202\265\030\004:\002ip' + _GEOIP.fields_by_name['file_path']._options = None + _GEOIP.fields_by_name['file_path']._serialized_options = b'\202\265\030\0172\rresource_hash' + _GEOSITE.fields_by_name['file_path']._options = None + _GEOSITE.fields_by_name['file_path']._serialized_options = b'\202\265\030\0172\rresource_hash' + _globals['_FIELDOPT']._serialized_start=62 + _globals['_FIELDOPT']._serialized_end=270 + _globals['_DOMAIN']._serialized_start=273 + _globals['_DOMAIN']._serialized_end=528 + _globals['_DOMAIN_ATTRIBUTE']._serialized_start=390 + _globals['_DOMAIN_ATTRIBUTE']._serialized_end=472 + _globals['_DOMAIN_TYPE']._serialized_start=474 + _globals['_DOMAIN_TYPE']._serialized_end=528 + _globals['_CIDR']._serialized_start=530 + _globals['_CIDR']._serialized_end=593 + _globals['_GEOIP']._serialized_start=596 + _globals['_GEOIP']._serialized_end=761 + _globals['_GEOIPLIST']._serialized_start=763 + _globals['_GEOIPLIST']._serialized_end=810 + _globals['_GEOSITE']._serialized_start=813 + _globals['_GEOSITE']._serialized_end=961 + _globals['_GEOSITELIST']._serialized_start=963 + _globals['_GEOSITELIST']._serialized_end=1014 +# @@protoc_insertion_point(module_scope) diff --git a/raydata.py b/raydata.py new file mode 100755 index 00000000..9550f6db --- /dev/null +++ b/raydata.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import typing +import sys +import os +from pathlib import Path +from ipaddress import IPv4Network, IPv6Network, AddressValueError +import ray_pb2 + + +def load_networks( + dat_files: typing.List[str], country_code: str +) -> typing.List[IPv4Network | IPv6Network]: + networks = [] + for file in dat_files: + file_path = Path(file).resolve() + if not file_path.exists() or not file_path.is_file(): + continue + data = ray_pb2.GeoIPList() + with open(file_path, 'rb') as fp: + data.ParseFromString(fp.read()) + for entry in data.entry: + if entry.country_code.lower() != country_code.lower(): + continue + for cidr in entry.cidr: + try: + networks.append(IPv4Network((cidr.ip, cidr.prefix))) + except AddressValueError: + pass + try: + networks.append(IPv6Network((cidr.ip, cidr.prefix))) + except AddressValueError: + pass + return networks + + +def load_domains( + dat_files: typing.List[str], country_code: str, domain_type: int | None = None +) -> typing.List[str]: + domains = [] + for file in dat_files: + file_path = Path(file).resolve() + if not file_path.exists() or not file_path.is_file(): + continue + data = ray_pb2.GeoSiteList() + with open(file_path, 'rb') as fp: + data.ParseFromString(fp.read()) + for entry in data.entry: + if entry.country_code.lower() != country_code.lower(): + continue + for domain in entry.domain: + if domain_type is None: + domains.append(domain.value) + elif domain.type == domain_type: + domains.append(domain.value) + return domains + + +def main(args): + input = getattr(args, 'input', None) + output = getattr(args, 'output', None) + + data_list = [] + if getattr(args, 'data_type', None) == 'geoip': + data_list = load_networks(input, getattr(args, 'country', None)) + if getattr(args, 'data_type', None) == 'geosite': + data_list = load_domains( + input, getattr(args, 'country', None), getattr(args, 'domain_type', None) + ) + if len(data_list) == 0: + print('No legal data.') + sys.exit(1) + output_lines = [ + f"{getattr(args,'prefix','')}{str(entry)}{getattr(args,'suffix','')}" + for entry in data_list + ] + if output is None: + for line in output_lines: + print(line, end='') + else: + output: Path + output_file = output.resolve() + if not output_file.parent.exists(): + os.makedirs(output_file.parent) + with output_file.open('w', encoding='utf-8') as wfp: + wfp.writelines(output_lines) + print(f'Output: {len(output_lines)}') + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + 'input', + nargs='+', + type=str, + default=[], + help='input ray data files (separated by space)', + ) + parser.add_argument('-c', '--country', type=str, default='', help='country code') + parser.add_argument('-o', '--output', type=Path, help='output file to write') + parser.add_argument( + '-p', '--prefix', type=str, default='', help='write line prefix' + ) + parser.add_argument( + '-s', '--suffix', type=str, default='\n', help='write line suffix' + ) + parser.set_defaults(func=main) + + subparser = parser.add_subparsers(dest='data_type', required=True) + + parser_geoip = subparser.add_parser( + 'geoip', + help='use geoip as input', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser_geosite = subparser.add_parser( + 'geosite', + help='use geosite as input', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser_geosite.add_argument( + '-t', + '--domain-type', + type=int, + help='geosite domain type', + ) + + args = parser.parse_args() + args.func(args) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4fc03022 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +aggregate-prefixes +protobuf