From 4b6287baed9606c2dd75863cccf0d344a8a36dcc Mon Sep 17 00:00:00 2001 From: darkclip Date: Fri, 21 Jul 2023 07:44:30 +0800 Subject: [PATCH] update --- .github/workflows/build-mosdns.yml | 28 ++--- .github/workflows/build-net.yml | 44 ++++--- .github/workflows/build-sha.yml | 30 ++--- exp_net.sh | 19 --- exp_ros.sh | 24 ++-- raydata.py | 180 ++++++++++++----------------- 6 files changed, 145 insertions(+), 180 deletions(-) delete mode 100755 exp_net.sh diff --git a/.github/workflows/build-mosdns.yml b/.github/workflows/build-mosdns.yml index 83ba3453..42380c1e 100644 --- a/.github/workflows/build-mosdns.yml +++ b/.github/workflows/build-mosdns.yml @@ -1,13 +1,13 @@ -name: build mosdns +name: build-mosdns on: push: branches: - 'master' paths: - - '.github/workflows/**' + - '.github/workflows/build-mosdns.yml' schedule: - - cron: '10 16 * * *' + - cron: '0 16 * * *' jobs: build-mosdns: @@ -24,28 +24,28 @@ jobs: run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Check out release branch of loyalsoldier/v2ray-rules-dat + - name: Check out reference branch uses: actions/checkout@v3 with: repository: loyalsoldier/v2ray-rules-dat ref: release - path: dat-release - - name: Checkout sub branch + path: ref + - name: Checkout target branch uses: actions/checkout@v3 with: ref: mosdns - path: mosdns - - name: Clean sub branch + path: target + - name: Clean target branch continue-on-error: true - run: rm mosdns/* - - name: Rebuild sub branch + run: rm target/* + - name: Rebuild target branch run: | - ./raydata.py dat-release/geoip.dat -c cn -o mosdns/chinaip.txt geoip - ./raydata.py dat-release/geosite.dat -c cn -o mosdns/chinasite.txt geosite --formatter format_mosdns.json - cd mosdns + ./raydata.py ref/geoip.dat -c cn -o target/chinaip.txt geoip + ./raydata.py ref/geosite.dat -c cn -o target/chinasite.txt geosite --formatter format_mosdns.json + cd target tar -cf mosdns.tar *.txt rm *.txt - - name: Push to sub branch + - name: Push to target branch continue-on-error: true run: | git config --global user.name "${GITHUB_ACTOR}" diff --git a/.github/workflows/build-net.yml b/.github/workflows/build-net.yml index 75813120..552a922a 100644 --- a/.github/workflows/build-net.yml +++ b/.github/workflows/build-net.yml @@ -1,11 +1,11 @@ -name: build net +name: build-net on: push: branches: - 'master' paths: - - '.github/workflows/**' + - '.github/workflows/build-net.yml' schedule: - cron: '0 16 * * *' @@ -24,27 +24,45 @@ jobs: run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Check out ip-lists branch of gaoyifan/china-operator-ip + - name: Check out reference branch uses: actions/checkout@v3 with: repository: gaoyifan/china-operator-ip ref: ip-lists - path: ip-lists - - name: Checkout sub branch + path: ref + - name: Checkout target branch uses: actions/checkout@v3 with: ref: net - path: net - - name: Clean sub branch + path: target + - name: Clean target branch continue-on-error: true - run: rm net/* - - name: Rebuild sub branch - run: ./exp_net.sh - - name: Push to sub branch + run: rm target/* + - name: Rebuild target branch + run: | + ./netgen.py ref/china.txt -o target/china.txt + ./netgen.py ref/china6.txt -o target/china6.txt + + ./netgen.py ref/chinanet.txt -o target/telecom.txt + ./netgen.py ref/chinanet6.txt -o target/telecom6.txt + + ./netgen.py ref/unicom.txt -o target/unicom.txt + ./netgen.py ref/unicom6.txt -o target/unicom6.txt + + ./netgen.py ref/china.txt -e ref/chinanet.txt ref/unicom.txt -o target/others.txt + ./netgen.py ref/china6.txt -e ref/chinanet6.txt ref/unicom6.txt -o target/others6.txt + + ./netgen.py 0.0.0.0/0 -e private.txt ref/china.txt -o target/foreign.txt + ./netgen.py ::/0 -e private6.txt ref/china6.txt -o target/foreign6.txt + + ./netgen.py private.txt -o target/private.txt + ./netgen.py private6.txt -o target/private6.txt + + cd target + tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html + - name: Push to target branch continue-on-error: true run: | - cd net - tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html git config --global user.name "${GITHUB_ACTOR}" git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" git add . diff --git a/.github/workflows/build-sha.yml b/.github/workflows/build-sha.yml index fd1483f7..4b669efa 100644 --- a/.github/workflows/build-sha.yml +++ b/.github/workflows/build-sha.yml @@ -1,13 +1,13 @@ -name: build sha +name: build-sha on: push: branches: - 'master' paths: - - '.github/workflows/**' + - '.github/workflows/build-sha.yml' schedule: - - cron: '20 16 * * *' + - cron: '0 16 * * *' jobs: build-sha: @@ -24,29 +24,31 @@ jobs: run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Check out ip-lists branch of gaoyifan/china-operator-ip + - name: Check out reference branch uses: actions/checkout@v3 with: repository: gaoyifan/china-operator-ip ref: ip-lists - path: ip-lists - - name: Checkout sub branch + path: ref + - name: Checkout target branch uses: actions/checkout@v3 with: ref: sha - path: sha - - name: Clean sub branch + path: target + - name: Clean target branch continue-on-error: true - run: rm sha/* - - name: Rebuild sub branch + run: rm target/* + - name: Rebuild target branch run: | - ./netgen.py ip-lists/china.txt -o sha/china.sh -p 'ip route add ' -s $' dev eth0 via 10.11.0.1 || true\n' - ./netgen.py ip-lists/china6.txt -o sha/china6.sh -p 'ip route add ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n' - cd sha + ./netgen.py ref/china.txt -o target/china.sh -p 'ip route add ' -s $' dev eth0 via 10.11.0.1 || true\n' + ./netgen.py ref/china6.txt -o target/china6.sh -p 'ip route add ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n' + ./netgen.py ref/china.txt -o target/rm-china.sh -p 'ip route del ' -s $' dev eth0 via 10.11.0.1 || true\n' + ./netgen.py ref/china6.txt -o target/rm-china6.sh -p 'ip route del ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n' + cd target chmod +x *.sh tar -cf sha.tar *.sh rm *.sh - - name: Push to sub branch + - name: Push to target branch continue-on-error: true run: | git config --global user.name "${GITHUB_ACTOR}" diff --git a/exp_net.sh b/exp_net.sh deleted file mode 100755 index cadea419..00000000 --- a/exp_net.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -./netgen.py ip-lists/china.txt -o net/china.txt -./netgen.py ip-lists/china6.txt -o net/china6.txt - -./netgen.py ip-lists/chinanet.txt -o net/telecom.txt -./netgen.py ip-lists/chinanet6.txt -o net/telecom6.txt - -./netgen.py ip-lists/unicom.txt -o net/unicom.txt -./netgen.py ip-lists/unicom6.txt -o net/unicom6.txt - -./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o net/others.txt -./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o net/others6.txt - -./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o net/foreign.txt -./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o net/foreign6.txt - -./netgen.py private.txt -o net/private.txt -./netgen.py private6.txt -o net/private6.txt diff --git a/exp_ros.sh b/exp_ros.sh index 2b6e7cac..3bf0bf28 100755 --- a/exp_ros.sh +++ b/exp_ros.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash -./netgen.py ip-lists/china.txt -o ros/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' -./netgen.py ip-lists/china6.txt -o ros/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' +./netgen.py ref/china.txt -o target/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' +./netgen.py ref/china6.txt -o target/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n' -./netgen.py ip-lists/chinanet.txt -o ros/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' -./netgen.py ip-lists/chinanet6.txt -o ros/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' +./netgen.py ref/chinanet.txt -o target/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' +./netgen.py ref/chinanet6.txt -o target/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n' -./netgen.py ip-lists/unicom.txt -o ros/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' -./netgen.py ip-lists/unicom6.txt -o ros/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' +./netgen.py ref/unicom.txt -o target/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' +./netgen.py ref/unicom6.txt -o target/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n' -./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o ros/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' -./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o ros/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' +./netgen.py ref/china.txt -e ref/chinanet.txt ref/unicom.txt -o target/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' +./netgen.py ref/china6.txt -e ref/chinanet6.txt ref/unicom6.txt -o target/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n' -./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o ros/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' -./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o ros/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' +./netgen.py 0.0.0.0/0 -e private.txt ref/china.txt -o target/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' +./netgen.py ::/0 -e private6.txt ref/china6.txt -o target/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n' -./netgen.py private.txt -o ros/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' -./netgen.py private6.txt -o ros/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' +./netgen.py private.txt -o target/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' +./netgen.py private6.txt -o target/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n' diff --git a/raydata.py b/raydata.py index a8451eaf..87051aea 100755 --- a/raydata.py +++ b/raydata.py @@ -11,103 +11,47 @@ from ipaddress import IPv4Network, IPv6Network, AddressValueError import ray_pb2 -def load_networks( - dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False -) -> typing.List[IPv4Network | IPv6Network]: - networks = [] - country_codes = [code.lower() for code in countries] - for file in dat_files: - file_path = file.resolve() - if not file_path.exists() or not file_path.is_file(): - continue - data = ray_pb2.GeoIPList() - with open(file_path, 'rb') as fp: - data.ParseFromString(fp.read()) - for entry in data.entry: - if inverse: - if entry.country_code.lower() in country_codes: - continue - else: - if entry.country_code.lower() not in country_codes: - continue - for cidr in entry.cidr: - try: - networks.append(IPv4Network((cidr.ip, cidr.prefix))) - except AddressValueError: - pass - try: - networks.append(IPv6Network((cidr.ip, cidr.prefix))) - except AddressValueError: - pass - return networks - - -def load_domains( - dat_files: typing.List[Path], - countries: typing.List[str], - inverse: bool = False, - formatter: Path | None = None, - domain_types: typing.List[int] | None = None, -) -> typing.List[str]: - domains = [] - country_codes = [code.lower() for code in countries] - format_dict = None - if formatter: +def load_formatter(formatter: Path | None) -> dict | None: + try: format_file_path = formatter.resolve() - if format_file_path.is_file(): - with format_file_path.open(encoding='utf-8') as fp: - format_dict = json.load(fp) - for file in dat_files: - file_path = file.resolve() - if not file_path.exists() or not file_path.is_file(): - continue - data = ray_pb2.GeoSiteList() - with open(file_path, 'rb') as fp: - data.ParseFromString(fp.read()) - for entry in data.entry: - if inverse: - if entry.country_code.lower() in country_codes: - continue - else: - if entry.country_code.lower() not in country_codes: - continue - for domain in entry.domain: - value = None - prefix = '' - suffix = '' - if format_dict: - prefix = format_dict.get('prefix').get(str(domain.type)) - suffix = format_dict.get('suffix').get(str(domain.type)) - if domain_types is None or domain.type in domain_types: - value = domain.value - if value is not None: - domains.append(f'{prefix}{domain.value}{suffix}') - return domains + with format_file_path.open(encoding='utf-8') as fp: + return json.load(fp) + except (AttributeError, OSError, json.JSONDecodeError, TypeError): + pass + return None -def show_data( - data_type: str, +def init_protobuf(data_type: str = None) -> tuple: + if data_type == 'geoip': + return (ray_pb2.GeoIPList(), 'cidr') + if data_type == 'geosite': + return (ray_pb2.GeoSiteList(), 'domain') + return (None, None) + + +def load_resource( dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False, -) -> None: + data_type: str = None, + *, + only_show: bool = False, + domain_types: typing.List[int] | None = None, + formatter: Path | None = None, +) -> list: + resources = [] country_codes = [code.lower() for code in countries] + format_dict = load_formatter(formatter) for file in dat_files: file_path = file.resolve() if not file_path.exists() or not file_path.is_file(): continue - data = None - if data_type == 'geoip': - data = ray_pb2.GeoIPList() - list_name = 'cidr' - if data_type == 'geosite': - data = ray_pb2.GeoSiteList() - list_name = 'domain' - if not data: - return + data_collection, list_name = init_protobuf(data_type) + if not data_collection: + return resources with open(file_path, 'rb') as fp: - data.ParseFromString(fp.read()) - for entry in data.entry: + data_collection.ParseFromString(fp.read()) + for entry in data_collection.entry: if len(country_codes) > 0: if inverse: if entry.country_code.lower() in country_codes: @@ -115,7 +59,32 @@ def show_data( else: if entry.country_code.lower() not in country_codes: continue - print(f"{entry.country_code}: {len(getattr(entry, list_name, None))}") + if only_show: + print( + f"{entry.country_code}: {sum(map(lambda record: not domain_types or record.type in domain_types, getattr(entry, list_name, None)))}" + ) + continue + for record in getattr(entry, list_name, None): + if list_name == 'cidr': + try: + resources.append(IPv4Network((record.ip, record.prefix))) + except AddressValueError: + pass + try: + resources.append(IPv6Network((record.ip, record.prefix))) + except AddressValueError: + pass + if list_name == 'domain': + if domain_types and record.type not in domain_types: + continue + prefix = '' + suffix = '' + if format_dict: + prefix = format_dict.get('prefix').get(str(record.type)) + suffix = format_dict.get('suffix').get(str(record.type)) + if record.value: + resources.append(f'{prefix}{record.value}{suffix}') + return resources def main(args): @@ -123,29 +92,24 @@ def main(args): output = getattr(args, 'output', None) if getattr(args, 'show', False): - show_data( - getattr(args, 'data_type', None), + load_resource( input_files, getattr(args, 'country', []), getattr(args, 'inverse', False), + getattr(args, 'data_type', None), + domain_types=getattr(args, 'domain_types', None), + only_show=True, ) sys.exit(0) - data_list = [] - if getattr(args, 'data_type', None) == 'geoip': - data_list = load_networks( - input_files, - getattr(args, 'country', []), - getattr(args, 'inverse', False), - ) - if getattr(args, 'data_type', None) == 'geosite': - data_list = load_domains( - input_files, - getattr(args, 'country', []), - getattr(args, 'inverse', None), - getattr(args, 'formatter', None), - getattr(args, 'domain_types', None), - ) + data_list = load_resource( + input_files, + getattr(args, 'country', []), + getattr(args, 'inverse', False), + getattr(args, 'data_type', None), + formatter=getattr(args, 'formatter', None), + domain_types=getattr(args, 'domain_types', None), + ) if len(data_list) == 0: print('No legal data.') sys.exit(1) @@ -218,17 +182,17 @@ if __name__ == '__main__': help='use geosite as input', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser_geosite.add_argument( - '--formatter', - type=Path, - help='format file for domain types', - ) parser_geosite.add_argument( '--domain-types', nargs='+', type=int, help='select domain types (separated by space)', ) + parser_geosite.add_argument( + '--formatter', + type=Path, + help='json formatter file base on domain types', + ) args = parser.parse_args() args.func(args)