diff --git a/format_mosdns.json b/format_mosdns.json new file mode 100644 index 00000000..d9c52881 --- /dev/null +++ b/format_mosdns.json @@ -0,0 +1,14 @@ +{ + "prefix": { + "0": "keyword:", + "1": "regexp:", + "2": "domain:", + "3": "full:" + }, + "suffix": { + "0": "", + "1": "", + "2": "", + "3": "" + } +} diff --git a/netgen.py b/netgen.py index e7c06397..ab1332f4 100755 --- a/netgen.py +++ b/netgen.py @@ -15,16 +15,16 @@ except ImportError: def prepare_networks( - nets_or_files: typing.List[str], only4: bool = False, only6: bool = False + nets_or_files: typing.List[str], family: typing.List[int] | None = None ) -> typing.List[IPv4Network | IPv6Network]: networks = [] for net in nets_or_files: - if not only6: + if not family or 4 in family: try: networks.append(IPv4Network(net)) except AddressValueError: pass - if not only4: + if not family or 6 in family: try: networks.append(IPv6Network(net)) except AddressValueError: @@ -35,12 +35,12 @@ def prepare_networks( continue with file_path.open(encoding='utf-8') as fp: for line in fp: - if not only6: + if not family or 4 in family: try: networks.append(IPv4Network(line.strip())) except AddressValueError: pass - if not only4: + if not family or 6 in family: try: networks.append(IPv6Network(line.strip())) except AddressValueError: @@ -65,15 +65,14 @@ def find_updated_networks( def main(args): includes = getattr(args, 'include', []) excludes = getattr(args, 'exclude', []) - only4 = getattr(args, 'ipv4', False) - only6 = getattr(args, 'ipv6', False) + family = getattr(args, 'family', None) output = getattr(args, 'output', None) - in_networks = prepare_networks(includes) + in_networks = prepare_networks(includes, family) if len(in_networks) == 0: print('No legal include ip networks.') sys.exit(1) - ex_networks = prepare_networks(excludes) + ex_networks = prepare_networks(excludes, family) out_networks = in_networks.copy() for exclude in ex_networks: out_networks = find_updated_networks(out_networks, exclude) @@ -81,9 +80,9 @@ def main(args): out4 = [] out6 = [] for net in out_networks: - if not only6 and isinstance(net, IPv4Network): + if (not family or 4 in family) and isinstance(net, IPv4Network): out4.append(net) - if not only4 and isinstance(net, IPv6Network): + if (not family or 6 in family) and isinstance(net, IPv6Network): out6.append(net) out4 = aggregate_prefixes(out4) out6 = aggregate_prefixes(out6) @@ -131,8 +130,20 @@ if __name__ == '__main__': default=[], help='exclude networks (subnets or files separated by space)', ) - parser.add_argument('-4', '--ipv4', action='store_true', help='process ipv4 only') - parser.add_argument('-6', '--ipv6', action='store_true', help='process ipv6 only') + parser.add_argument( + '-4', + dest='family', + action='append_const', + const=4, + help='process ipv4 only', + ) + parser.add_argument( + '-6', + dest='family', + action='append_const', + const=6, + help='process ipv6 only', + ) parser.add_argument('-o', '--output', type=Path, help='output file to write') parser.add_argument( '-p', '--prefix', type=str, default='', help='write line prefix' diff --git a/raydata.py b/raydata.py index c470ddf4..be1518d8 100755 --- a/raydata.py +++ b/raydata.py @@ -5,18 +5,19 @@ from __future__ import annotations import typing import sys import os +import json from pathlib import Path from ipaddress import IPv4Network, IPv6Network, AddressValueError import ray_pb2 def load_networks( - dat_files: typing.List[str], countries: typing.List[str], inverse: bool = False + dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False ) -> typing.List[IPv4Network | IPv6Network]: networks = [] country_codes = [code.lower() for code in countries] for file in dat_files: - file_path = Path(file).resolve() + file_path = file.resolve() if not file_path.exists() or not file_path.is_file(): continue data = ray_pb2.GeoIPList() @@ -42,15 +43,22 @@ def load_networks( def load_domains( - dat_files: typing.List[str], + dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False, - domain_type: int | None = None, + formatter: Path | None = None, + domain_types: typing.List[int] | None = None, ) -> typing.List[str]: domains = [] country_codes = [code.lower() for code in countries] + format_dict = None + if formatter: + format_file_path = formatter.resolve() + if format_file_path.is_file(): + with format_file_path.open(encoding='utf-8') as fp: + format_dict = json.load(fp) for file in dat_files: - file_path = Path(file).resolve() + file_path = file.resolve() if not file_path.exists() or not file_path.is_file(): continue data = ray_pb2.GeoSiteList() @@ -65,32 +73,36 @@ def load_domains( continue for domain in entry.domain: value = None - if domain_type is None: + prefix = '' + suffix = '' + if format_dict: + prefix = format_dict.get('prefix').get(str(domain.type)) + suffix = format_dict.get('suffix').get(str(domain.type)) + if domain_types is None or domain.type in domain_types: value = domain.value - elif domain.type == domain_type: - value = domain.value - if value: - domains.append(domain.value) + if value is not None: + domains.append(f'{prefix}{domain.value}{suffix}') return domains def main(args): - input_files = getattr(args, 'input', None) + input_files = getattr(args, 'input', []) output = getattr(args, 'output', None) data_list = [] if getattr(args, 'data_type', None) == 'geoip': data_list = load_networks( input_files, - getattr(args, 'countries', None), - getattr(args, 'inverse', None), + getattr(args, 'country', []), + getattr(args, 'inverse', False), ) if getattr(args, 'data_type', None) == 'geosite': data_list = load_domains( input_files, - getattr(args, 'countries', None), + getattr(args, 'country', []), getattr(args, 'inverse', None), - getattr(args, 'domain_type', None), + getattr(args, 'formatter', None), + getattr(args, 'domain_types', None), ) if len(data_list) == 0: print('No legal data.') @@ -121,17 +133,16 @@ if __name__ == '__main__': parser.add_argument( 'input', nargs='+', - type=str, + type=Path, default=[], help='input ray data files (separated by space)', ) parser.add_argument( '-c', - '--countries', - nargs='+', + '--country', type=str, - default=[], - help='country codes (separated by space)', + action='append', + help='country codes (use multiple times)', ) parser.add_argument( '-i', @@ -162,9 +173,15 @@ if __name__ == '__main__': formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser_geosite.add_argument( - '--domain-type', + '--formatter', + type=Path, + help='format file for domain types', + ) + parser_geosite.add_argument( + '--domain-types', + nargs='+', type=int, - help='geosite domain type', + help='select domain types (separated by space)', ) args = parser.parse_args()