update

.github/workflows/build-net.yml

@@ -0,0 +1,50 @@
+name: build net
+
+on:
+  push:
+    branches: ['*']
+  schedule:
+    - cron: '0 16 * * *'
+
+jobs:
+  build-net:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Check out ip-lists branch of gaoyifan/china-operator-ip
+        uses: actions/checkout@v3
+        with:
+          repository: gaoyifan/china-operator-ip
+          ref: ip-lists
+          path: ip-lists
+      - name: Checkout net branch
+        uses: actions/checkout@v3
+        with:
+          ref: net
+          path: net
+      - name: Rebuild net branch
+        if: github.event_name == 'schedule'
+        run: |
+          rm net/*
+          ./exp_net.sh
+      - name: Push to net branch
+        if: github.event_name == 'schedule'
+        continue-on-error: true
+        run: |
+          cd net
+          tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git add .
+          git commit -m "update $(date +%Y-%m-%d)"
+          git push -q

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+/venv

exp_net.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+./netgen.py ip-lists/china.txt -o net/china.txt
+./netgen.py ip-lists/china6.txt -o net/china6.txt
+
+./netgen.py ip-lists/chinanet.txt -o net/telecom.txt
+./netgen.py ip-lists/chinanet6.txt -o net/telecom6.txt
+
+./netgen.py ip-lists/unicom.txt -o net/unicom.txt
+./netgen.py ip-lists/unicom6.txt -o net/unicom6.txt
+
+./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o net/others.txt
+./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o net/others6.txt
+
+./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o net/foreign.txt
+./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o net/foreign6.txt
+
+./netgen.py private.txt -o net/private.txt
+./netgen.py private6.txt -o net/private6.txt

exp_ros.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+./netgen.py ip-lists/china.txt -o ros/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
+./netgen.py ip-lists/china6.txt -o ros/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
+
+./netgen.py ip-lists/chinanet.txt -o ros/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
+./netgen.py ip-lists/chinanet6.txt -o ros/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
+
+./netgen.py ip-lists/unicom.txt -o ros/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
+./netgen.py ip-lists/unicom6.txt -o ros/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
+
+./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o ros/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
+./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o ros/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
+
+./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o ros/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
+./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o ros/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
+
+./netgen.py private.txt -o ros/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'
+./netgen.py private6.txt -o ros/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'

netgen.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import typing
+import sys
+import os
+from pathlib import Path
+from ipaddress import IPv4Network, IPv6Network, AddressValueError
+
+try:
+    from aggregate_prefixes import aggregate_prefixes
+except ImportError:
+    aggregate_prefixes = None
+
+
+def prepare_networks(
+    nets_or_files: typing.List[str], only4: bool = False, only6: bool = False
+) -> typing.List[IPv4Network | IPv6Network]:
+    networks = []
+    for net in nets_or_files:
+        if not only6:
+            try:
+                networks.append(IPv4Network(net))
+            except AddressValueError:
+                pass
+        if not only4:
+            try:
+                networks.append(IPv6Network(net))
+            except AddressValueError:
+                pass
+    for file in nets_or_files:
+        file_path = Path(file).resolve()
+        if not file_path.exists() or not file_path.is_file():
+            continue
+        with file_path.open(encoding='utf-8') as fp:
+            for line in fp:
+                if not only6:
+                    try:
+                        networks.append(IPv4Network(line.strip()))
+                    except AddressValueError:
+                        pass
+                if not only4:
+                    try:
+                        networks.append(IPv6Network(line.strip()))
+                    except AddressValueError:
+                        pass
+    return networks
+
+
+def find_updated_networks(
+    networks: typing.List[IPv4Network | IPv6Network], subnet: IPv4Network | IPv6Network
+) -> typing.List[IPv4Network | IPv6Network]:
+    copied_networks = networks.copy()
+    for index, network in enumerate(networks):
+        try:
+            if subnet.subnet_of(network):
+                copied_networks.pop(index)
+                copied_networks.extend(list(network.address_exclude(subnet)))
+        except TypeError:
+            pass
+    return copied_networks
+
+
+def main(args):
+    includes = getattr(args, 'include', [])
+    excludes = getattr(args, 'exclude', [])
+    only4 = getattr(args, 'ipv4', False)
+    only6 = getattr(args, 'ipv6', False)
+    output = getattr(args, 'output', None)
+
+    in_networks = prepare_networks(includes)
+    if len(in_networks) == 0:
+        print('No legal include ip networks.')
+        sys.exit(1)
+    ex_networks = prepare_networks(excludes)
+    out_networks = in_networks.copy()
+    for exclude in ex_networks:
+        out_networks = find_updated_networks(out_networks, exclude)
+    if aggregate_prefixes is not None:
+        out4 = []
+        out6 = []
+        for net in out_networks:
+            if not only6 and isinstance(net, IPv4Network):
+                out4.append(net)
+            if not only4 and isinstance(net, IPv6Network):
+                out6.append(net)
+        out4 = aggregate_prefixes(out4)
+        out6 = aggregate_prefixes(out6)
+        out_networks = []
+        out_networks.extend(out4)
+        out_networks.extend(out6)
+
+    output_lines = [
+        f"{getattr(args,'prefix','')}{str(network)}{getattr(args,'suffix','')}"
+        for network in out_networks
+    ]
+
+    if output is None:
+        for line in output_lines:
+            print(line, end='')
+    else:
+        print(f'Included Networks: {len(in_networks)}')
+        output: Path
+        output_file = output.resolve()
+        if not output_file.parent.exists():
+            os.makedirs(output_file.parent)
+        with output_file.open('w', encoding='utf-8') as wfp:
+            wfp.writelines(output_lines)
+        print(f'Generated Networks: {len(output_lines)}')
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        'include',
+        nargs='+',
+        type=str,
+        default=[],
+        help='include networks (subnets or files separated by space)',
+    )
+    parser.add_argument(
+        '-e',
+        '--exclude',
+        nargs='+',
+        type=str,
+        default=[],
+        help='exclude networks (subnets or files separated by space)',
+    )
+    parser.add_argument('-4', '--ipv4', action='store_true', help='process ipv4 only')
+    parser.add_argument('-6', '--ipv6', action='store_true', help='process ipv6 only')
+    parser.add_argument('-o', '--output', type=Path, help='output file to write')
+    parser.add_argument(
+        '-p', '--prefix', type=str, default='', help='write line prefix'
+    )
+    parser.add_argument(
+        '-s', '--suffix', type=str, default='\n', help='write line suffix'
+    )
+    parser.set_defaults(func=main)
+    args = parser.parse_args()
+    args.func(args)

private.txt

@@ -0,0 +1,8 @@
+10.0.0.0/8
+172.16.0.0/12
+192.168.0.0/16
+127.0.0.0/8
+169.254.0.0/16
+198.18.0.0/15
+224.0.0.0/4
+240.0.0.0/4

private6.txt

@@ -0,0 +1,4 @@
+::1/128
+fc00::/7
+fe80::/10
+ff00::/8

ray.proto

@@ -0,0 +1,102 @@
+// source: app/router/routercommon/common.proto
+// source: common/protoext/extensions.proto
+// generate: protoc --python_out=./ ray.proto
+
+syntax = "proto3";
+package routercommon;
+import "google/protobuf/descriptor.proto";
+
+message FieldOpt {
+  repeated string any_wants = 1;
+  repeated string allowed_values = 2;
+  repeated string allowed_value_types = 3;
+
+  // convert_time_read_file_into read a file into another field, and clear this
+  // field during input parsing
+  string convert_time_read_file_into = 4;
+  // forbidden marks a boolean to be inaccessible to user
+  bool forbidden = 5;
+  // convert_time_resource_loading read a file, and place its resource hash into
+  // another field
+  string convert_time_resource_loading = 6;
+  // convert_time_parse_ip parse a string ip address, and put its binary
+  // representation into another field
+  string convert_time_parse_ip = 7;
+}
+
+extend google.protobuf.FieldOptions { FieldOpt field_opt = 50000; }
+
+// Domain for routing decision.
+message Domain {
+  // Type of domain value.
+  enum Type {
+    // The value is used as is.
+    Plain = 0;
+    // The value is used as a regular expression.
+    Regex = 1;
+    // The value is a root domain.
+    RootDomain = 2;
+    // The value is a domain.
+    Full = 3;
+  }
+
+  // Domain matching type.
+  Type type = 1;
+
+  // Domain value.
+  string value = 2;
+
+  message Attribute {
+    string key = 1;
+
+    oneof typed_value {
+      bool bool_value = 2;
+      int64 int_value = 3;
+    }
+  }
+
+  // Attributes of this domain. May be used for filtering.
+  repeated Attribute attribute = 3;
+}
+
+// IP for routing decision, in CIDR form.
+message CIDR {
+  // IP address, should be either 4 or 16 bytes.
+  bytes ip = 1;
+
+  // Number of leading ones in the network mask.
+  uint32 prefix = 2;
+
+  string ip_addr = 68000 [ (field_opt).convert_time_parse_ip = "ip" ];
+}
+
+message GeoIP {
+  string country_code = 1;
+  repeated CIDR cidr = 2;
+  bool inverse_match = 3;
+
+  // resource_hash instruct simplified config converter to load domain from geo
+  // file.
+  bytes resource_hash = 4;
+  string code = 5;
+
+  string file_path = 68000
+      [ (field_opt).convert_time_resource_loading = "resource_hash" ];
+}
+
+message GeoIPList { repeated GeoIP entry = 1; }
+
+message GeoSite {
+  string country_code = 1;
+  repeated Domain domain = 2;
+
+  // resource_hash instruct simplified config converter to load domain from geo
+  // file.
+  bytes resource_hash = 3;
+  string code = 4;
+
+  string file_path = 68000
+      [ (field_opt).convert_time_resource_loading = "resource_hash" ];
+}
+
+message GeoSiteList { repeated GeoSite entry = 1; }

ray_pb2.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: ray.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tray.proto\x12\x0croutercommon\x1a google/protobuf/descriptor.proto\"\xd0\x01\n\x08\x46ieldOpt\x12\x11\n\tany_wants\x18\x01 \x03(\t\x12\x16\n\x0e\x61llowed_values\x18\x02 \x03(\t\x12\x1b\n\x13\x61llowed_value_types\x18\x03 \x03(\t\x12#\n\x1b\x63onvert_time_read_file_into\x18\x04 \x01(\t\x12\x11\n\tforbidden\x18\x05 \x01(\x08\x12%\n\x1d\x63onvert_time_resource_loading\x18\x06 \x01(\t\x12\x1d\n\x15\x63onvert_time_parse_ip\x18\x07 \x01(\t\"\xff\x01\n\x06\x44omain\x12\'\n\x04type\x18\x01 \x01(\x0e\x32\x19.routercommon.Domain.Type\x12\r\n\x05value\x18\x02 \x01(\t\x12\x31\n\tattribute\x18\x03 \x03(\x0b\x32\x1e.routercommon.Domain.Attribute\x1aR\n\tAttribute\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x42\r\n\x0btyped_value\"6\n\x04Type\x12\t\n\x05Plain\x10\x00\x12\t\n\x05Regex\x10\x01\x12\x0e\n\nRootDomain\x10\x02\x12\x08\n\x04\x46ull\x10\x03\"?\n\x04\x43IDR\x12\n\n\x02ip\x18\x01 \x01(\x0c\x12\x0e\n\x06prefix\x18\x02 \x01(\r\x12\x1b\n\x07ip_addr\x18\xa0\x93\x04 \x01(\tB\x08\x82\xb5\x18\x04:\x02ip\"\xa5\x01\n\x05GeoIP\x12\x14\n\x0c\x63ountry_code\x18\x01 \x01(\t\x12 \n\x04\x63idr\x18\x02 \x03(\x0b\x32\x12.routercommon.CIDR\x12\x15\n\rinverse_match\x18\x03 \x01(\x08\x12\x15\n\rresource_hash\x18\x04 \x01(\x0c\x12\x0c\n\x04\x63ode\x18\x05 \x01(\t\x12(\n\tfile_path\x18\xa0\x93\x04 \x01(\tB\x13\x82\xb5\x18\x0f\x32\rresource_hash\"/\n\tGeoIPList\x12\"\n\x05\x65ntry\x18\x01 \x03(\x0b\x32\x13.routercommon.GeoIP\"\x94\x01\n\x07GeoSite\x12\x14\n\x0c\x63ountry_code\x18\x01 \x01(\t\x12$\n\x06\x64omain\x18\x02 \x03(\x0b\x32\x14.routercommon.Domain\x12\x15\n\rresource_hash\x18\x03 \x01(\x0c\x12\x0c\n\x04\x63ode\x18\x04 \x01(\t\x12(\n\tfile_path\x18\xa0\x93\x04 \x01(\tB\x13\x82\xb5\x18\x0f\x32\rresource_hash\"3\n\x0bGeoSiteList\x12$\n\x05\x65ntry\x18\x01 \x03(\x0b\x32\x15.routercommon.GeoSite:J\n\tfield_opt\x12\x1d.google.protobuf.FieldOptions\x18\xd0\x86\x03 \x01(\x0b\x32\x16.routercommon.FieldOptb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ray_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(field_opt)
+
+  DESCRIPTOR._options = None
+  _CIDR.fields_by_name['ip_addr']._options = None
+  _CIDR.fields_by_name['ip_addr']._serialized_options = b'\202\265\030\004:\002ip'
+  _GEOIP.fields_by_name['file_path']._options = None
+  _GEOIP.fields_by_name['file_path']._serialized_options = b'\202\265\030\0172\rresource_hash'
+  _GEOSITE.fields_by_name['file_path']._options = None
+  _GEOSITE.fields_by_name['file_path']._serialized_options = b'\202\265\030\0172\rresource_hash'
+  _globals['_FIELDOPT']._serialized_start=62
+  _globals['_FIELDOPT']._serialized_end=270
+  _globals['_DOMAIN']._serialized_start=273
+  _globals['_DOMAIN']._serialized_end=528
+  _globals['_DOMAIN_ATTRIBUTE']._serialized_start=390
+  _globals['_DOMAIN_ATTRIBUTE']._serialized_end=472
+  _globals['_DOMAIN_TYPE']._serialized_start=474
+  _globals['_DOMAIN_TYPE']._serialized_end=528
+  _globals['_CIDR']._serialized_start=530
+  _globals['_CIDR']._serialized_end=593
+  _globals['_GEOIP']._serialized_start=596
+  _globals['_GEOIP']._serialized_end=761
+  _globals['_GEOIPLIST']._serialized_start=763
+  _globals['_GEOIPLIST']._serialized_end=810
+  _globals['_GEOSITE']._serialized_start=813
+  _globals['_GEOSITE']._serialized_end=961
+  _globals['_GEOSITELIST']._serialized_start=963
+  _globals['_GEOSITELIST']._serialized_end=1014
+# @@protoc_insertion_point(module_scope)

raydata.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import typing
+import sys
+import os
+from pathlib import Path
+from ipaddress import IPv4Network, IPv6Network, AddressValueError
+import ray_pb2
+
+
+def load_networks(
+    dat_files: typing.List[str], country_code: str
+) -> typing.List[IPv4Network | IPv6Network]:
+    networks = []
+    for file in dat_files:
+        file_path = Path(file).resolve()
+        if not file_path.exists() or not file_path.is_file():
+            continue
+        data = ray_pb2.GeoIPList()
+        with open(file_path, 'rb') as fp:
+            data.ParseFromString(fp.read())
+        for entry in data.entry:
+            if entry.country_code.lower() != country_code.lower():
+                continue
+            for cidr in entry.cidr:
+                try:
+                    networks.append(IPv4Network((cidr.ip, cidr.prefix)))
+                except AddressValueError:
+                    pass
+                try:
+                    networks.append(IPv6Network((cidr.ip, cidr.prefix)))
+                except AddressValueError:
+                    pass
+    return networks
+
+
+def load_domains(
+    dat_files: typing.List[str], country_code: str, domain_type: int | None = None
+) -> typing.List[str]:
+    domains = []
+    for file in dat_files:
+        file_path = Path(file).resolve()
+        if not file_path.exists() or not file_path.is_file():
+            continue
+        data = ray_pb2.GeoSiteList()
+        with open(file_path, 'rb') as fp:
+            data.ParseFromString(fp.read())
+        for entry in data.entry:
+            if entry.country_code.lower() != country_code.lower():
+                continue
+            for domain in entry.domain:
+                if domain_type is None:
+                    domains.append(domain.value)
+                elif domain.type == domain_type:
+                    domains.append(domain.value)
+    return domains
+
+
+def main(args):
+    input = getattr(args, 'input', None)
+    output = getattr(args, 'output', None)
+
+    data_list = []
+    if getattr(args, 'data_type', None) == 'geoip':
+        data_list = load_networks(input, getattr(args, 'country', None))
+    if getattr(args, 'data_type', None) == 'geosite':
+        data_list = load_domains(
+            input, getattr(args, 'country', None), getattr(args, 'domain_type', None)
+        )
+    if len(data_list) == 0:
+        print('No legal data.')
+        sys.exit(1)
+    output_lines = [
+        f"{getattr(args,'prefix','')}{str(entry)}{getattr(args,'suffix','')}"
+        for entry in data_list
+    ]
+    if output is None:
+        for line in output_lines:
+            print(line, end='')
+    else:
+        output: Path
+        output_file = output.resolve()
+        if not output_file.parent.exists():
+            os.makedirs(output_file.parent)
+        with output_file.open('w', encoding='utf-8') as wfp:
+            wfp.writelines(output_lines)
+        print(f'Output: {len(output_lines)}')
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        'input',
+        nargs='+',
+        type=str,
+        default=[],
+        help='input ray data files (separated by space)',
+    )
+    parser.add_argument('-c', '--country', type=str, default='', help='country code')
+    parser.add_argument('-o', '--output', type=Path, help='output file to write')
+    parser.add_argument(
+        '-p', '--prefix', type=str, default='', help='write line prefix'
+    )
+    parser.add_argument(
+        '-s', '--suffix', type=str, default='\n', help='write line suffix'
+    )
+    parser.set_defaults(func=main)
+
+    subparser = parser.add_subparsers(dest='data_type', required=True)
+
+    parser_geoip = subparser.add_parser(
+        'geoip',
+        help='use geoip as input',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser_geosite = subparser.add_parser(
+        'geosite',
+        help='use geosite as input',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser_geosite.add_argument(
+        '-t',
+        '--domain-type',
+        type=int,
+        help='geosite domain type',
+    )
+
+    args = parser.parse_args()
+    args.func(args)

requirements.txt

@@ -0,0 +1,2 @@
+aggregate-prefixes
+protobuf