Build V2Ray rules dat files #340
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build V2Ray rules dat files | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 22 * * *" | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "**/README.md" | |
| permissions: | |
| contents: write | |
| env: | |
| EXTRA_EXPORT_FILES: "netflix youtube primevideo hbo disney roku spotify tiktok twitch xvideos pornhub dmm dlsite telegram instagram threads facebook snap speedtest openai playstation nintendo xbox steam ea epicgames ubisoft" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set variables | |
| run: | | |
| echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV | |
| echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf" >> $GITHUB_ENV | |
| echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV | |
| echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV | |
| echo "EASYLISTCHINA_EASYLIST_REJECT_URL=https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt" >> $GITHUB_ENV | |
| echo "PETERLOWE_REJECT_URL=https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext" >> $GITHUB_ENV | |
| echo "ADGUARD_DNS_REJECT_URL=https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" >> $GITHUB_ENV | |
| echo "DANPOLLOCK_REJECT_URL=https://someonewhocares.org/hosts/hosts" >> $GITHUB_ENV | |
| echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/daiyeqi/domain-list-custom/release/cn.txt" >> $GITHUB_ENV | |
| echo "CUSTOM_PROXY=https://raw.githubusercontent.com/daiyeqi/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV | |
| echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV | |
| echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV | |
| echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV | |
| echo "GEOIP_HASH=https://raw.githubusercontent.com/daiyeqi/v2ray-rules-dat/release/geoip.dat.sha256sum" >> $GITHUB_ENV | |
| echo "GEOSITE_HASH=https://raw.githubusercontent.com/daiyeqi/v2ray-rules-dat/release/geosite.dat.sha256sum" >> $GITHUB_ENV | |
| echo "RULETAR_HASH=https://raw.githubusercontent.com/daiyeqi/v2ray-rules-dat/release/rules.tar.sha256sum" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Checkout the "hidden" branch of this repo | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: hidden | |
| - name: Checkout daiyeqi/domain-list-custom | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: daiyeqi/domain-list-custom | |
| path: custom | |
| - name: Checkout v2fly/domain-list-community | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: v2fly/domain-list-community | |
| path: community | |
| - name: Checkout cokebar/gfwlist2dnsmasq | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: cokebar/gfwlist2dnsmasq | |
| path: gfwlist2dnsmasq | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: ./custom/go.mod | |
| cache-dependency-path: ./custom/go.sum | |
| - name: Get geoip.dat relative files | |
| run: | | |
| wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat | |
| wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat.sha256sum | |
| - name: Generate GFWList domains | |
| run: | | |
| cd gfwlist2dnsmasq || exit 1 | |
| chmod +x ./gfwlist2dnsmasq.sh | |
| ./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt | |
| - name: Get and add direct domains into temp-direct.txt file | |
| run: | | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt | |
| curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-direct.txt | |
| - name: Get and add proxy domains into temp-proxy.txt file | |
| run: | | |
| cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-proxy.txt | |
| - name: Get and add reject domains into temp-reject.txt file | |
| run: | | |
| curl -sSL $EASYLISTCHINA_EASYLIST_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' > temp-reject.txt | |
| curl -sSL $ADGUARD_DNS_REJECT_URL | perl -ne '/^\|\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' | perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' >> temp-reject.txt | |
| curl -sSL $PETERLOWE_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})$/ && print "$1\n"' >> temp-reject.txt | |
| curl -sSL $DANPOLLOCK_REJECT_URL | perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})/ && print "$1\n"' | sed '1d' >> temp-reject.txt | |
| - name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files | |
| run: | | |
| curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt | |
| curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt | |
| - name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files | |
| run: | | |
| cat proxy.txt >> temp-proxy.txt | |
| cat direct.txt >> temp-direct.txt | |
| cat reject.txt >> temp-reject.txt | |
| - name: Sort and generate redundant lists | |
| run: | | |
| cat temp-proxy.txt | sort --ignore-case -u > proxy-list-with-redundant | |
| cat temp-direct.txt | sort --ignore-case -u > direct-list-with-redundant | |
| cat temp-reject.txt | sort --ignore-case -u > reject-list-with-redundant | |
| - name: Remove redundant domains | |
| run: | | |
| chmod +x findRedundantDomain.py | |
| ./findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort | |
| ./findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort | |
| ./findRedundantDomain.py ./reject-list-with-redundant ./reject-list-deleted-unsort | |
| [ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort | |
| [ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort | |
| [ ! -f "reject-list-deleted-unsort" ] && touch reject-list-deleted-unsort | |
| sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort | |
| sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort | |
| sort ./reject-list-deleted-unsort > ./reject-list-deleted-sort | |
| diff ./direct-list-deleted-sort ./direct-list-with-redundant | awk '/^>/{print $2}' > ./direct-list-without-redundant | |
| diff ./proxy-list-deleted-sort ./proxy-list-with-redundant | awk '/^>/{print $2}' > ./proxy-list-without-redundant | |
| diff ./reject-list-deleted-sort ./reject-list-with-redundant | awk '/^>/{print $2}' > ./reject-list-without-redundant | |
| - name: Remove domains from "need-to-remove" lists in "hidden" branch | |
| run: | | |
| diff ./direct-need-to-remove.txt ./direct-list-without-redundant | awk '/^>/{print $2}' > temp-cn.txt | |
| diff ./proxy-need-to-remove.txt ./proxy-list-without-redundant | awk '/^>/{print $2}' > temp-geolocation-\!cn.txt | |
| diff ./reject-need-to-remove.txt ./reject-list-without-redundant | awk '/^>/{print $2}' > temp-category-ads-all.txt | |
| - name: Write lists to data directory | |
| run: | | |
| cat temp-cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn | |
| cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt | |
| cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/geolocation-\!cn | |
| cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt | |
| cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/category-ads-all | |
| cat temp-category-ads-all.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > reject-tld-list.txt | |
| - name: Add `full`, `regexp` and `keyword` type of rules back into "cn", "geolocation-!cn" and "category-ads-all" list | |
| run: | | |
| [ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn | |
| [ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn | |
| [ -f "reject-reserve.txt" ] && cat reject-reserve.txt >> ./community/data/category-ads-all | |
| cp ./community/data/cn direct-list.txt | |
| cp ./community/data/geolocation-\!cn proxy-list.txt | |
| cp ./community/data/category-ads-all reject-list.txt | |
| - name: Create `china-list`、`google-cn`、`apple-cn`、`gfw`、`greatfire` lists | |
| run: | | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > ./community/data/china-list | |
| curl -sSL $CHINA_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > china-list.txt | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn | |
| curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > google-cn.txt | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn | |
| curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > apple-cn.txt | |
| cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw | |
| cat ./community/data/gfw | sort --ignore-case -u > gfw.txt | |
| cat ./community/data/greatfire | sort --ignore-case -u > greatfire.txt | |
| curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-spy | |
| curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > win-spy.txt | |
| curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-update | |
| curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > win-update.txt | |
| curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-extra | |
| curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > win-extra.txt | |
| - name: Build geosite.dat file | |
| run: | | |
| cd custom || exit 1 | |
| FLAG_EXTRA_EXPORT_FILES=$(echo "${{ env.EXTRA_EXPORT_FILES }}"| tr ' ' ',') | |
| go run ./ --datapath=../community/data --exportlists=$FLAG_EXTRA_EXPORT_FILES | |
| - name: Move and tar files and generate sha256 hash | |
| run: | | |
| install -Dp ./geoip.dat ./publish/geoip.dat | |
| install -Dp ./geoip.dat.sha256sum ./publish/geoip.dat.sha256sum | |
| install -Dp ./custom/publish/geosite.dat ./publish/geosite.dat | |
| for export_file in ${{ env.EXTRA_EXPORT_FILES }}; do | |
| cat ./custom/publish/${export_file}.txt | sort --ignore-case -u > ${export_file}.txt | |
| install -p ${export_file}.txt ./publish/ | |
| done | |
| install -p {proxy,direct,reject}-tld-list.txt ./publish/ | |
| install -p {proxy,direct,reject}-list.txt ./publish/ | |
| install -p {china-list,apple-cn,google-cn,gfw,greatfire,win-spy,win-update,win-extra}.txt ./publish/ | |
| cd publish || exit 1 | |
| sha256sum geosite.dat > geosite.dat.sha256sum | |
| tar --sort=name --mtime='1970-01-01 00:00:00Z' -cf rules.tar *.txt | |
| sha256sum rules.tar > rules.tar.sha256sum | |
| - name: Compare build checksum against previous release | |
| run: | | |
| cd publish || exit 1 | |
| if ! diff -q geoip.dat.sha256sum <(curl -sL "${{ env.GEOIP_HASH }}"); then | |
| echo "NEED_RELEASE=true" >> $GITHUB_ENV | |
| elif ! diff -q geosite.dat.sha256sum <(curl -sL "${{ env.GEOSITE_HASH }}"); then | |
| echo "NEED_RELEASE=true" >> $GITHUB_ENV | |
| elif ! diff -q rules.tar.sha256sum <(curl -sL "${{ env.RULETAR_HASH }}"); then | |
| echo "NEED_RELEASE=true" >> $GITHUB_ENV | |
| fi | |
| - name: Git push assets to "release" branch | |
| if: ${{ env.NEED_RELEASE }} | |
| run: | | |
| cd publish || exit 1 | |
| git init | |
| git config --local user.name "github-actions[bot]" | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git checkout -b release | |
| git add . | |
| git commit -m "Released on ${{ env.TAG_NAME }}" | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f -u origin release | |
| - name: Cleanup releases and tags | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| KEEP_RELEASES: 5 | |
| if: ${{ env.NEED_RELEASE }} | |
| run: | | |
| gh release list --repo "${{ github.repository }}" --limit 1000 --json tagName --jq ".[$KEEP_RELEASES:] | .[].tagName" | xargs -r -n1 gh release delete --repo "${{ github.repository }}" --cleanup-tag --yes | |
| shell: bash | |
| - name: Release and upload assets | |
| if: ${{ env.NEED_RELEASE }} | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: ${{ env.TAG_NAME }} | |
| tag: ${{ env.TAG_NAME }} | |
| file_glob: true | |
| file: ./publish/*.{dat,tar,sha256sum} |