update

.github/workflows/build-mosdns.yml

@@ -1,13 +1,13 @@
-name: build mosdns
+name: build-mosdns
 
 on:
   push:
     branches:
       - 'master'
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/build-mosdns.yml'
   schedule:
-    - cron: '10 16 * * *'
+    - cron: '0 16 * * *'
 
 jobs:
   build-mosdns:
@@ -24,28 +24,28 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Check out release branch of loyalsoldier/v2ray-rules-dat
+      - name: Check out reference branch
         uses: actions/checkout@v3
         with:
           repository: loyalsoldier/v2ray-rules-dat
           ref: release
-          path: dat-release
-      - name: Checkout sub branch
+          path: ref
+      - name: Checkout target branch
         uses: actions/checkout@v3
         with:
           ref: mosdns
-          path: mosdns
-      - name: Clean sub branch
+          path: target
+      - name: Clean target branch
         continue-on-error: true
-        run: rm mosdns/*
-      - name: Rebuild sub branch
+        run: rm target/*
+      - name: Rebuild target branch
         run: |
-          ./raydata.py dat-release/geoip.dat -c cn -o mosdns/chinaip.txt geoip
-          ./raydata.py dat-release/geosite.dat -c cn -o mosdns/chinasite.txt geosite --formatter format_mosdns.json
-          cd mosdns
+          ./raydata.py ref/geoip.dat -c cn -o target/chinaip.txt geoip
+          ./raydata.py ref/geosite.dat -c cn -o target/chinasite.txt geosite --formatter format_mosdns.json
+          cd target
           tar -cf mosdns.tar *.txt
           rm *.txt
-      - name: Push to sub branch
+      - name: Push to target branch
         continue-on-error: true
         run: |
           git config --global user.name "${GITHUB_ACTOR}"

.github/workflows/build-net.yml

@@ -1,11 +1,11 @@
-name: build net
+name: build-net
 
 on:
   push:
     branches:
       - 'master'
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/build-net.yml'
   schedule:
     - cron: '0 16 * * *'
 
@@ -24,27 +24,45 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Check out ip-lists branch of gaoyifan/china-operator-ip
+      - name: Check out reference branch
         uses: actions/checkout@v3
         with:
           repository: gaoyifan/china-operator-ip
           ref: ip-lists
-          path: ip-lists
-      - name: Checkout sub branch
+          path: ref
+      - name: Checkout target branch
         uses: actions/checkout@v3
         with:
           ref: net
-          path: net
-      - name: Clean sub branch
+          path: target
+      - name: Clean target branch
         continue-on-error: true
-        run: rm net/*
-      - name: Rebuild sub branch
-        run: ./exp_net.sh
-      - name: Push to sub branch
+        run: rm target/*
+      - name: Rebuild target branch
+        run: |
+          ./netgen.py ref/china.txt -o target/china.txt
+          ./netgen.py ref/china6.txt -o target/china6.txt
+
+          ./netgen.py ref/chinanet.txt -o target/telecom.txt
+          ./netgen.py ref/chinanet6.txt -o target/telecom6.txt
+
+          ./netgen.py ref/unicom.txt -o target/unicom.txt
+          ./netgen.py ref/unicom6.txt -o target/unicom6.txt
+
+          ./netgen.py ref/china.txt -e ref/chinanet.txt ref/unicom.txt -o target/others.txt
+          ./netgen.py ref/china6.txt -e ref/chinanet6.txt ref/unicom6.txt -o target/others6.txt
+
+          ./netgen.py 0.0.0.0/0 -e private.txt ref/china.txt -o target/foreign.txt
+          ./netgen.py ::/0 -e private6.txt ref/china6.txt -o target/foreign6.txt
+
+          ./netgen.py private.txt -o target/private.txt
+          ./netgen.py private6.txt -o target/private6.txt
+
+          cd target
+          tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html
+      - name: Push to target branch
         continue-on-error: true
         run: |
-          cd net
-          tree -H . -P "*.txt" -T "IP List - $(date +%Y-%m-%d)" > index.html
           git config --global user.name "${GITHUB_ACTOR}"
           git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
           git add .

.github/workflows/build-sha.yml

@@ -1,13 +1,13 @@
-name: build sha
+name: build-sha
 
 on:
   push:
     branches:
       - 'master'
     paths:
-      - '.github/workflows/**'
+      - '.github/workflows/build-sha.yml'
   schedule:
-    - cron: '20 16 * * *'
+    - cron: '0 16 * * *'
 
 jobs:
   build-sha:
@@ -24,29 +24,31 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Check out ip-lists branch of gaoyifan/china-operator-ip
+      - name: Check out reference branch
         uses: actions/checkout@v3
         with:
           repository: gaoyifan/china-operator-ip
           ref: ip-lists
-          path: ip-lists
-      - name: Checkout sub branch
+          path: ref
+      - name: Checkout target branch
         uses: actions/checkout@v3
         with:
           ref: sha
-          path: sha
-      - name: Clean sub branch
+          path: target
+      - name: Clean target branch
         continue-on-error: true
-        run: rm sha/*
-      - name: Rebuild sub branch
+        run: rm target/*
+      - name: Rebuild target branch
         run: |
-          ./netgen.py ip-lists/china.txt -o sha/china.sh -p 'ip route add ' -s $' dev eth0 via 10.11.0.1 || true\n'
-          ./netgen.py ip-lists/china6.txt -o sha/china6.sh -p 'ip route add ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n'
-          cd sha
+          ./netgen.py ref/china.txt -o target/china.sh -p 'ip route add ' -s $' dev eth0 via 10.11.0.1 || true\n'
+          ./netgen.py ref/china6.txt -o target/china6.sh -p 'ip route add ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n'
+          ./netgen.py ref/china.txt -o target/rm-china.sh -p 'ip route del ' -s $' dev eth0 via 10.11.0.1 || true\n'
+          ./netgen.py ref/china6.txt -o target/rm-china6.sh -p 'ip route del ' -s $' dev eth0 via fe80::6a77:24ff:fed6:809d || true\n'
+          cd target
           chmod +x *.sh
           tar -cf sha.tar *.sh
           rm *.sh
-      - name: Push to sub branch
+      - name: Push to target branch
         continue-on-error: true
         run: |
           git config --global user.name "${GITHUB_ACTOR}"

exp_net.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-./netgen.py ip-lists/china.txt -o net/china.txt
-./netgen.py ip-lists/china6.txt -o net/china6.txt
-
-./netgen.py ip-lists/chinanet.txt -o net/telecom.txt
-./netgen.py ip-lists/chinanet6.txt -o net/telecom6.txt
-
-./netgen.py ip-lists/unicom.txt -o net/unicom.txt
-./netgen.py ip-lists/unicom6.txt -o net/unicom6.txt
-
-./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o net/others.txt
-./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o net/others6.txt
-
-./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o net/foreign.txt
-./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o net/foreign6.txt
-
-./netgen.py private.txt -o net/private.txt
-./netgen.py private6.txt -o net/private6.txt

exp_ros.sh

@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 
-./netgen.py ip-lists/china.txt -o ros/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
-./netgen.py ip-lists/china6.txt -o ros/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
+./netgen.py ref/china.txt -o target/china.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
+./netgen.py ref/china6.txt -o target/china6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_China\n'
 
-./netgen.py ip-lists/chinanet.txt -o ros/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
-./netgen.py ip-lists/chinanet6.txt -o ros/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
+./netgen.py ref/chinanet.txt -o target/telecom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
+./netgen.py ref/chinanet6.txt -o target/telecom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaTelecom\n'
 
-./netgen.py ip-lists/unicom.txt -o ros/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
-./netgen.py ip-lists/unicom6.txt -o ros/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
+./netgen.py ref/unicom.txt -o target/unicom.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
+./netgen.py ref/unicom6.txt -o target/unicom6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaUnicom\n'
 
-./netgen.py ip-lists/china.txt -e ip-lists/chinanet.txt ip-lists/unicom.txt -o ros/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
-./netgen.py ip-lists/china6.txt -e ip-lists/chinanet6.txt ip-lists/unicom6.txt -o ros/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
+./netgen.py ref/china.txt -e ref/chinanet.txt ref/unicom.txt -o target/others.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
+./netgen.py ref/china6.txt -e ref/chinanet6.txt ref/unicom6.txt -o target/others6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_ChinaOthers\n'
 
-./netgen.py 0.0.0.0/0 -e private.txt ip-lists/china.txt -o ros/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
-./netgen.py ::/0 -e private6.txt ip-lists/china6.txt -o ros/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
+./netgen.py 0.0.0.0/0 -e private.txt ref/china.txt -o target/foreign.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
+./netgen.py ::/0 -e private6.txt ref/china6.txt -o target/foreign6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Foreign\n'
 
-./netgen.py private.txt -o ros/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'
-./netgen.py private6.txt -o ros/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'
+./netgen.py private.txt -o target/private.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'
+./netgen.py private6.txt -o target/private6.rsc -p '/routing rule add dst-address=' -s $' action=lookup table=Route_Private\n'

raydata.py

@@ -11,103 +11,47 @@ from ipaddress import IPv4Network, IPv6Network, AddressValueError
 import ray_pb2
 
 
-def load_networks(
-    dat_files: typing.List[Path], countries: typing.List[str], inverse: bool = False
-) -> typing.List[IPv4Network | IPv6Network]:
-    networks = []
-    country_codes = [code.lower() for code in countries]
-    for file in dat_files:
-        file_path = file.resolve()
-        if not file_path.exists() or not file_path.is_file():
-            continue
-        data = ray_pb2.GeoIPList()
-        with open(file_path, 'rb') as fp:
-            data.ParseFromString(fp.read())
-        for entry in data.entry:
-            if inverse:
-                if entry.country_code.lower() in country_codes:
-                    continue
-            else:
-                if entry.country_code.lower() not in country_codes:
-                    continue
-            for cidr in entry.cidr:
-                try:
-                    networks.append(IPv4Network((cidr.ip, cidr.prefix)))
-                except AddressValueError:
-                    pass
-                try:
-                    networks.append(IPv6Network((cidr.ip, cidr.prefix)))
-                except AddressValueError:
-                    pass
-    return networks
-
-
-def load_domains(
-    dat_files: typing.List[Path],
-    countries: typing.List[str],
-    inverse: bool = False,
-    formatter: Path | None = None,
-    domain_types: typing.List[int] | None = None,
-) -> typing.List[str]:
-    domains = []
-    country_codes = [code.lower() for code in countries]
-    format_dict = None
-    if formatter:
+def load_formatter(formatter: Path | None) -> dict | None:
+    try:
         format_file_path = formatter.resolve()
-        if format_file_path.is_file():
-            with format_file_path.open(encoding='utf-8') as fp:
-                format_dict = json.load(fp)
-    for file in dat_files:
-        file_path = file.resolve()
-        if not file_path.exists() or not file_path.is_file():
-            continue
-        data = ray_pb2.GeoSiteList()
-        with open(file_path, 'rb') as fp:
-            data.ParseFromString(fp.read())
-        for entry in data.entry:
-            if inverse:
-                if entry.country_code.lower() in country_codes:
-                    continue
-            else:
-                if entry.country_code.lower() not in country_codes:
-                    continue
-            for domain in entry.domain:
-                value = None
-                prefix = ''
-                suffix = ''
-                if format_dict:
-                    prefix = format_dict.get('prefix').get(str(domain.type))
-                    suffix = format_dict.get('suffix').get(str(domain.type))
-                if domain_types is None or domain.type in domain_types:
-                    value = domain.value
-                if value is not None:
-                    domains.append(f'{prefix}{domain.value}{suffix}')
-    return domains
+        with format_file_path.open(encoding='utf-8') as fp:
+            return json.load(fp)
+    except (AttributeError, OSError, json.JSONDecodeError, TypeError):
+        pass
+    return None
 
 
-def show_data(
-    data_type: str,
+def init_protobuf(data_type: str = None) -> tuple:
+    if data_type == 'geoip':
+        return (ray_pb2.GeoIPList(), 'cidr')
+    if data_type == 'geosite':
+        return (ray_pb2.GeoSiteList(), 'domain')
+    return (None, None)
+
+
+def load_resource(
     dat_files: typing.List[Path],
     countries: typing.List[str],
     inverse: bool = False,
-) -> None:
+    data_type: str = None,
+    *,
+    only_show: bool = False,
+    domain_types: typing.List[int] | None = None,
+    formatter: Path | None = None,
+) -> list:
+    resources = []
     country_codes = [code.lower() for code in countries]
+    format_dict = load_formatter(formatter)
     for file in dat_files:
         file_path = file.resolve()
         if not file_path.exists() or not file_path.is_file():
             continue
-        data = None
-        if data_type == 'geoip':
-            data = ray_pb2.GeoIPList()
-            list_name = 'cidr'
-        if data_type == 'geosite':
-            data = ray_pb2.GeoSiteList()
-            list_name = 'domain'
-        if not data:
-            return
+        data_collection, list_name = init_protobuf(data_type)
+        if not data_collection:
+            return resources
         with open(file_path, 'rb') as fp:
-            data.ParseFromString(fp.read())
-        for entry in data.entry:
+            data_collection.ParseFromString(fp.read())
+        for entry in data_collection.entry:
             if len(country_codes) > 0:
                 if inverse:
                     if entry.country_code.lower() in country_codes:
@@ -115,7 +59,32 @@ def show_data(
                 else:
                     if entry.country_code.lower() not in country_codes:
                         continue
-            print(f"{entry.country_code}: {len(getattr(entry, list_name, None))}")
+            if only_show:
+                print(
+                    f"{entry.country_code}: {sum(map(lambda record: not domain_types or record.type in domain_types, getattr(entry, list_name, None)))}"
+                )
+                continue
+            for record in getattr(entry, list_name, None):
+                if list_name == 'cidr':
+                    try:
+                        resources.append(IPv4Network((record.ip, record.prefix)))
+                    except AddressValueError:
+                        pass
+                    try:
+                        resources.append(IPv6Network((record.ip, record.prefix)))
+                    except AddressValueError:
+                        pass
+                if list_name == 'domain':
+                    if domain_types and record.type not in domain_types:
+                        continue
+                    prefix = ''
+                    suffix = ''
+                    if format_dict:
+                        prefix = format_dict.get('prefix').get(str(record.type))
+                        suffix = format_dict.get('suffix').get(str(record.type))
+                    if record.value:
+                        resources.append(f'{prefix}{record.value}{suffix}')
+    return resources
 
 
 def main(args):
@@ -123,29 +92,24 @@ def main(args):
     output = getattr(args, 'output', None)
 
     if getattr(args, 'show', False):
-        show_data(
-            getattr(args, 'data_type', None),
+        load_resource(
             input_files,
             getattr(args, 'country', []),
             getattr(args, 'inverse', False),
+            getattr(args, 'data_type', None),
+            domain_types=getattr(args, 'domain_types', None),
+            only_show=True,
         )
         sys.exit(0)
 
-    data_list = []
-    if getattr(args, 'data_type', None) == 'geoip':
-        data_list = load_networks(
-            input_files,
-            getattr(args, 'country', []),
-            getattr(args, 'inverse', False),
-        )
-    if getattr(args, 'data_type', None) == 'geosite':
-        data_list = load_domains(
-            input_files,
-            getattr(args, 'country', []),
-            getattr(args, 'inverse', None),
-            getattr(args, 'formatter', None),
-            getattr(args, 'domain_types', None),
-        )
+    data_list = load_resource(
+        input_files,
+        getattr(args, 'country', []),
+        getattr(args, 'inverse', False),
+        getattr(args, 'data_type', None),
+        formatter=getattr(args, 'formatter', None),
+        domain_types=getattr(args, 'domain_types', None),
+    )
     if len(data_list) == 0:
         print('No legal data.')
         sys.exit(1)
@@ -218,17 +182,17 @@ if __name__ == '__main__':
         help='use geosite as input',
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
-    parser_geosite.add_argument(
-        '--formatter',
-        type=Path,
-        help='format file for domain types',
-    )
     parser_geosite.add_argument(
         '--domain-types',
         nargs='+',
         type=int,
         help='select domain types (separated by space)',
     )
+    parser_geosite.add_argument(
+        '--formatter',
+        type=Path,
+        help='json formatter file base on domain types',
+    )
 
     args = parser.parse_args()
     args.func(args)