This commit is contained in:
darkclip committed 2023-07-20 03:14:26 +08:00
1 parent 7c1ca5ffe0
commit ae219614a3
3 files changed
+77 -35

No files matched your search

+14
View File
@@ -0,0 +1,14 @@
{
"prefix": {
"0": "keyword:",
"1": "regexp:",
"2": "domain:",
"3": "full:"
},
"suffix": {
"0": "",
"1": "",
"2": "",
"3": ""
}
}
+24 -13
View File
@@ -15,16 +15,16 @@ except ImportError:
def prepare_networks(
nets_or_files: typing.List[str], only4: bool = False, only6: bool = False
nets_or_files: typing.List[str], family: typing.List[int] | None = None
) -> typing.List[IPv4Network | IPv6Network]:
networks = []
for net in nets_or_files:
if not only6:
if not family or 4 in family:
try:
networks.append(IPv4Network(net))
except AddressValueError:
pass
if not only4:
if not family or 6 in family:
try:
networks.append(IPv6Network(net))
except AddressValueError:
@@ -35,12 +35,12 @@ def prepare_networks(
continue
with file_path.open(encoding='utf-8') as fp:
for line in fp:
if not only6:
if not family or 4 in family:
try:
networks.append(IPv4Network(line.strip()))
except AddressValueError:
pass
if not only4:
if not family or 6 in family:
try:
networks.append(IPv6Network(line.strip()))
except AddressValueError:
@@ -65,15 +65,14 @@ def find_updated_networks(
def main(args):
includes = getattr(args, 'include', [])
excludes = getattr(args, 'exclude', [])
only4 = getattr(args, 'ipv4', False)
only6 = getattr(args, 'ipv6', False)
family = getattr(args, 'family', None)
output = getattr(args, 'output', None)
in_networks = prepare_networks(includes)
in_networks = prepare_networks(includes, family)
if len(in_networks) == 0:
print('No legal include ip networks.')
sys.exit(1)
ex_networks = prepare_networks(excludes)
ex_networks = prepare_networks(excludes, family)
out_networks = in_networks.copy()
for exclude in ex_networks:
out_networks = find_updated_networks(out_networks, exclude)
@@ -81,9 +80,9 @@ def main(args):
out4 = []
out6 = []
for net in out_networks:
if not only6 and isinstance(net, IPv4Network):
if (not family or 4 in family) and isinstance(net, IPv4Network):
out4.append(net)
if not only4 and isinstance(net, IPv6Network):
if (not family or 6 in family) and isinstance(net, IPv6Network):
out6.append(net)
out4 = aggregate_prefixes(out4)
out6 = aggregate_prefixes(out6)
@@ -131,8 +130,20 @@ if __name__ == '__main__':
default=[],
help='exclude networks (subnets or files separated by space)',
)
parser.add_argument('-4', '--ipv4', action='store_true', help='process ipv4 only')
parser.add_argument('-6', '--ipv6', action='store_true', help='process ipv6 only')
parser.add_argument(
'-4',
dest='family',
action='append_const',
const=4,
help='process ipv4 only',
)
parser.add_argument(
'-6',
dest='family',
action='append_const',
const=6,
help='process ipv6 only',
)
parser.add_argument('-o', '--output', type=Path, help='output file to write')
parser.add_argument(
'-p', '--prefix', type=str, default='', help='write line prefix'
+39 -22
View File
@@ -5,18 +5,19 @@ from __future__ import annotations
import typing
import sys
import os
import json
from pathlib import Path
from ipaddress import IPv4Network, IPv6Network, AddressValueError
import ray_pb2
def load_networks(
dat_files: typing.List[str], countries: typing.List[str], inverse: bool = False
dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False
) -> typing.List[IPv4Network | IPv6Network]:
networks = []
country_codes = [code.lower() for code in countries]
for file in dat_files:
file_path = Path(file).resolve()
file_path = file.resolve()
if not file_path.exists() or not file_path.is_file():
continue
data = ray_pb2.GeoIPList()
@@ -42,15 +43,22 @@ def load_networks(
def load_domains(
dat_files: typing.List[str],
dat_files: typing.List[Path],
countries: typing.List[str],
inverse: bool = False,
domain_type: int | None = None,
formatter: Path | None = None,
domain_types: typing.List[int] | None = None,
) -> typing.List[str]:
domains = []
country_codes = [code.lower() for code in countries]
format_dict = None
if formatter:
format_file_path = formatter.resolve()
if format_file_path.is_file():
with format_file_path.open(encoding='utf-8') as fp:
format_dict = json.load(fp)
for file in dat_files:
file_path = Path(file).resolve()
file_path = file.resolve()
if not file_path.exists() or not file_path.is_file():
continue
data = ray_pb2.GeoSiteList()
@@ -65,32 +73,36 @@ def load_domains(
continue
for domain in entry.domain:
value = None
if domain_type is None:
prefix = ''
suffix = ''
if format_dict:
prefix = format_dict.get('prefix').get(str(domain.type))
suffix = format_dict.get('suffix').get(str(domain.type))
if domain_types is None or domain.type in domain_types:
value = domain.value
elif domain.type == domain_type:
value = domain.value
if value:
domains.append(domain.value)
if value is not None:
domains.append(f'{prefix}{domain.value}{suffix}')
return domains
def main(args):
input_files = getattr(args, 'input', None)
input_files = getattr(args, 'input', [])
output = getattr(args, 'output', None)
data_list = []
if getattr(args, 'data_type', None) == 'geoip':
data_list = load_networks(
input_files,
getattr(args, 'countries', None),
getattr(args, 'inverse', None),
getattr(args, 'country', []),
getattr(args, 'inverse', False),
)
if getattr(args, 'data_type', None) == 'geosite':
data_list = load_domains(
input_files,
getattr(args, 'countries', None),
getattr(args, 'country', []),
getattr(args, 'inverse', None),
getattr(args, 'domain_type', None),
getattr(args, 'formatter', None),
getattr(args, 'domain_types', None),
)
if len(data_list) == 0:
print('No legal data.')
@@ -121,17 +133,16 @@ if __name__ == '__main__':
parser.add_argument(
'input',
nargs='+',
type=str,
type=Path,
default=[],
help='input ray data files (separated by space)',
)
parser.add_argument(
'-c',
'--countries',
nargs='+',
'--country',
type=str,
default=[],
help='country codes (separated by space)',
action='append',
help='country codes (use multiple times)',
)
parser.add_argument(
'-i',
@@ -162,9 +173,15 @@ if __name__ == '__main__':
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser_geosite.add_argument(
'--domain-type',
'--formatter',
type=Path,
help='format file for domain types',
)
parser_geosite.add_argument(
'--domain-types',
nargs='+',
type=int,
help='geosite domain type',
help='select domain types (separated by space)',
)
args = parser.parse_args()