From f0f382c8781646f8c6d385adf03d0b194a17850c Mon Sep 17 00:00:00 2001 From: mr00huang <13051806099@163.com> Date: Wed, 22 Jan 2025 09:24:29 +0000 Subject: [PATCH 1/2] add tor config v2 --- Dockerfile | 13 ++-- dockerfiles/Dockerfile_debian12 | 3 +- dockerfiles/Dockerfile_ubuntu20 | 3 +- dockerfiles/Dockerfile_ubuntu24 | 3 +- src/spider_traffic/torDo.py | 104 ++++++++++++++++++++++++++++++++ torrc_s/torrc_meek | 13 ++++ torrc_s/torrc_obfs4 | 15 +++++ torrc_s/torrc_obfs4_0 | 15 +++++ torrc_s/torrc_obfs4_1 | 15 +++++ torrc_s/torrc_obfs4_2 | 14 +++++ torrc_s/torrc_snowflake | 12 ++++ torrc_s/torrc_tor | 15 +++++ torrc_s/torrc_webtunnel | 17 ++++++ torrc_s/torrc_webtunnel_ipv6 | 17 ++++++ 14 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 src/spider_traffic/torDo.py create mode 100644 torrc_s/torrc_meek create mode 100644 torrc_s/torrc_obfs4 create mode 100644 torrc_s/torrc_obfs4_0 create mode 100644 torrc_s/torrc_obfs4_1 create mode 100644 torrc_s/torrc_obfs4_2 create mode 100644 torrc_s/torrc_snowflake create mode 100644 torrc_s/torrc_tor create mode 100644 torrc_s/torrc_webtunnel create mode 100644 torrc_s/torrc_webtunnel_ipv6 diff --git a/Dockerfile b/Dockerfile index c6955f1..5e0a948 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM 192.168.194.63:5000/ubuntu:latest +FROM ubuntu:latest # 设置工作目录 WORKDIR /app @@ -65,13 +65,14 @@ RUN sudo apt install -y \ libxfixes3 \ libxkbcommon0 \ libxrandr2 \ + libevent-dev \ xdg-utils RUN sudo apt install -y libasound2t64 -RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb -RUN sudo apt install -f -RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb -RUN sudo mv bin/chromedriver-linux64/chromedriver /usr/bin +#RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb +#RUN sudo apt install -f +#RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb +#RUN sudo mv bin/chromedriver-linux64/chromedriver /usr/bin # 默认命令，打开vim -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/dockerfiles/Dockerfile_debian12 b/dockerfiles/Dockerfile_debian12 index ccd5b9a..c9820a3 100644 --- a/dockerfiles/Dockerfile_debian12 +++ b/dockerfiles/Dockerfile_debian12 @@ -71,6 +71,7 @@ RUN sudo apt install -y \ libxfixes3 \ libxkbcommon0 \ libxrandr2 \ + libevent-dev \ xdg-utils RUN sudo apt install -y libasound2 RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb @@ -80,4 +81,4 @@ RUN sudo mv bin/chromedriver-linux64/chromedriver /usr/bin # 默认命令，打开vim -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/dockerfiles/Dockerfile_ubuntu20 b/dockerfiles/Dockerfile_ubuntu20 index 93b918e..cfb45f2 100644 --- a/dockerfiles/Dockerfile_ubuntu20 +++ b/dockerfiles/Dockerfile_ubuntu20 @@ -75,6 +75,7 @@ RUN sudo apt install -y \ libxfixes3 \ libxkbcommon0 \ libxrandr2 \ + libevent-dev \ xdg-utils RUN sudo apt install -y libasound2 RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb @@ -84,4 +85,4 @@ RUN sudo mv bin/chromedriver-linux64/chromedriver /usr/bin # 默认命令，打开vim -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/dockerfiles/Dockerfile_ubuntu24 b/dockerfiles/Dockerfile_ubuntu24 index c6955f1..88605e3 100644 --- a/dockerfiles/Dockerfile_ubuntu24 +++ b/dockerfiles/Dockerfile_ubuntu24 @@ -65,6 +65,7 @@ RUN sudo apt install -y \ libxfixes3 \ libxkbcommon0 \ libxrandr2 \ + libevent-dev \ xdg-utils RUN sudo apt install -y libasound2t64 RUN sudo dpkg -i bin/google-chrome-stable_current_amd64.deb @@ -74,4 +75,4 @@ RUN sudo mv bin/chromedriver-linux64/chromedriver /usr/bin # 默认命令，打开vim -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/src/spider_traffic/torDo.py b/src/spider_traffic/torDo.py new file mode 100644 index 0000000..2bb708b --- /dev/null +++ b/src/spider_traffic/torDo.py @@ -0,0 +1,104 @@ +import os +from multiprocessing import Process +import subprocess +import time + +from spider_traffic.myutils.logger import logger + + +BIN_FOLDER='/app/bin/tor_s' +TORRC_FOLDER='/app/torrc_s' +TOR_LOG_FOLDER='/app/logs' + + + + +def long_running_task(log_file): + while True: + time.sleep(1) + f = os.popen(f"cat {log_file}|grep 'Bootstrapped 100%'|wc -l") + process_num = int(f.read()) + if process_num != 0: + break + + +def wait_for_100(log_file,timeout=180): + # 创建子进程 + process = Process(target=long_running_task, args=(log_file,)) + process.start() + process.join(timeout=timeout) # 等待最多 x秒 + if process.is_alive(): + process.terminate() # 强制终止子进程 + return False + else: + return True + + + +def start_tor(flag:str): + """ + 根据flag启动Tor程序: + 'none':裸Tor + 'meek':使用Meek的Tor + 'obfs4_0'使用Obfs4且客户端iat-mode=0的Tor + 'obfs4_1'使用Obfs4且客户端iat-mode=1的Tor + 'obfs4_2'使用Obfs4且客户端iat-mode=2的Tor + 'snowflake'使用snowflake的Tor + 'webtunnel'使用webtunnel的Tor + Return: + 进程,启动成功标志(True为启动成功, False为启动不成功) + """ + #根据配置制作/挑选torrc文件 + if flag=='none': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_tor') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_none.log') + elif flag=='webtunnel': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_webtunnel') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_webtunnel.log') + elif flag=='meek': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_meek') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_meek.log') + elif flag=='snowflake': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_snowflake') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_snowflake.log') + elif flag=='obfs4_0': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_obfs4_0') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_obfs4_0.log') + elif flag=='obfs4_1': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_obfs4_1') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_obfs4_1.log') + elif flag=='obfs4_2': + torrc_file=os.path.join(TORRC_FOLDER,'torrc_obfs4_2') + log_file=os.path.join(TOR_LOG_FOLDER,'notice_obfs4_2.log') + else: + raise TypeError(f'Unknown Tor protocol flag!: {flag}') + #清空log日志用于后续判断Tor是否启动成功 + subprocess.call("echo > %s" % log_file, shell=True) + #使用torrc文件启动Tor + tor_process = subprocess.Popen( + [os.path.join(BIN_FOLDER,'tor'), "-f", torrc_file], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + #判断Tor是否启动成功 + tor_is_started=wait_for_100(log_file,300) + logger.info(f'Tor-{flag}启动成功? {tor_is_started}!!') + return tor_process,tor_is_started + +def close_tor(tor_process): + tor_process.terminate() # 尝试优雅地关闭进程 + # 如果进程没有退出，使用kill强制终止 + try: + tor_process.wait(timeout=5) # 等待进程退出，最多等5秒 + logger.info("优雅的关闭Tor进程") + except subprocess.TimeoutExpired: + tor_process.kill() # 如果进程没有在超时前退出，强制杀死进程 + logger.info("强制杀死Tor进程") + +if __name__=='__main__': + #测试启动Tor与关闭Tor + tor_process,success=start_tor('webtunnel') + print(f'{time.time}Tor启动成功?:{success}') + time.sleep(20) + close_tor(tor_process) diff --git a/torrc_s/torrc_meek b/torrc_s/torrc_meek new file mode 100644 index 0000000..68ebd5c --- /dev/null +++ b/torrc_s/torrc_meek @@ -0,0 +1,13 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_meek +Log notice file /app/logs/notice_meek.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 +ClientTransportPlugin meek exec /app/bin/tor_s/meek-client + +Bridge meek 0.0.2.0:3 url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com diff --git a/torrc_s/torrc_obfs4 b/torrc_s/torrc_obfs4 new file mode 100644 index 0000000..83a3e16 --- /dev/null +++ b/torrc_s/torrc_obfs4 @@ -0,0 +1,15 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData +Log notice file /app/logs/notice.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 + +ClientTransportPlugin obfs4 exec /app/bin/tor_s/obfs4proxy + +Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0 + diff --git a/torrc_s/torrc_obfs4_0 b/torrc_s/torrc_obfs4_0 new file mode 100644 index 0000000..cd94a83 --- /dev/null +++ b/torrc_s/torrc_obfs4_0 @@ -0,0 +1,15 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_obfs4_0 +Log notice file /app/logs/notice_obfs4_0.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 + +ClientTransportPlugin obfs4 exec /app/bin/tor_s/obfs4proxy + +Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0 + diff --git a/torrc_s/torrc_obfs4_1 b/torrc_s/torrc_obfs4_1 new file mode 100644 index 0000000..1d8aa68 --- /dev/null +++ b/torrc_s/torrc_obfs4_1 @@ -0,0 +1,15 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_obfs4_1 +Log notice file /app/logs/notice_obfs4_1.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 + +ClientTransportPlugin obfs4 exec /app/bin/tor_s/obfs4proxy + +Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=1 + diff --git a/torrc_s/torrc_obfs4_2 b/torrc_s/torrc_obfs4_2 new file mode 100644 index 0000000..f0baa72 --- /dev/null +++ b/torrc_s/torrc_obfs4_2 @@ -0,0 +1,14 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_obfs4_2 +Log notice file /app/logs/notice_obfs4_2.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 + +ClientTransportPlugin obfs4 exec /app/bin/tor_s/obfs4proxy + +Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=2 diff --git a/torrc_s/torrc_snowflake b/torrc_s/torrc_snowflake new file mode 100644 index 0000000..133527c --- /dev/null +++ b/torrc_s/torrc_snowflake @@ -0,0 +1,12 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_snowflake +Log notice file /app/logs/notice_snowflake.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 +ClientTransportPlugin snowflake exec /app/bin/tor_s/snowflake-client +Bridge snowflake 192.0.2.4:80 8838024498816A039FCBBAB14E6F40A0843051FA fingerprint=8838024498816A039FCBBAB14E6F40A0843051FA url=https://snowflake-broker.torproject.net.global.prod.fastly.net/ front=cdn.sstatic.net ice=stun:stun.l.google.com:19302,stun:stun.antisip.com:3478,stun:stun.bluesip.net:3478,stun:stun.dus.net:3478,stun:stun.epygi.com:3478,stun:stun.sonetel.net:3478,stun:stun.uls.co.za:3478,stun:stun.voipgate.com:3478,stun:stun.voys.nl:3478 utls-imitate=hellorandomizedalpn diff --git a/torrc_s/torrc_tor b/torrc_s/torrc_tor new file mode 100644 index 0000000..79baaf2 --- /dev/null +++ b/torrc_s/torrc_tor @@ -0,0 +1,15 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_none +Log notice file /app/logs/notice_none.log +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +#UseBridges 1 + +#ClientTransportPlugin obfs4 exec /app/bin/tor_s/obfs4proxy + +#Bridge obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0 + diff --git a/torrc_s/torrc_webtunnel b/torrc_s/torrc_webtunnel new file mode 100644 index 0000000..4dfd4a3 --- /dev/null +++ b/torrc_s/torrc_webtunnel @@ -0,0 +1,17 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_webtunnel +Log notice file /app/logs/notice_webtunnel.log + +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 +ClientTransportPlugin webtunnel exec /app/bin/tor_s/webtunnelClient + +Bridge webtunnel 99.121.60.23:443 3BDD7036FAC9CC81ADBA39E88637D5EF87350F1C url=https://speedtest.qs.ee/dI6HDfnDLSUiFV0MaLMEtcmv ver=0.0.1 + + + diff --git a/torrc_s/torrc_webtunnel_ipv6 b/torrc_s/torrc_webtunnel_ipv6 new file mode 100644 index 0000000..394cfdb --- /dev/null +++ b/torrc_s/torrc_webtunnel_ipv6 @@ -0,0 +1,17 @@ +ControlPort 2700 +SocksPort 9050 +DataDirectory /app/torData/tor_webtunnel +Log notice file /app/logs/notice_webtunnel.log + +SafeLogging 0 +RunAsDaemon 0 + +LogTimeGranularity 1 + +UseBridges 1 +ClientTransportPlugin webtunnel exec /app/bin/tor_s/webtunnelClient + +Bridge webtunnel [2001:db8:afc8:4fe0:c021:5377:36bf:e9b7]:443 3BDD7036FAC9CC81ADBA39E88637D5EF87350F1C url=https://speedtest.qs.ee/dI6HDfnDLSUiFV0MaLMEtcmv ver=0.0.1 + + + From 01339aaabd0574db955fdac75431b94b350096d3 Mon Sep 17 00:00:00 2001 From: mr00huang <13051806099@163.com> Date: Wed, 22 Jan 2025 09:28:53 +0000 Subject: [PATCH 2/2] add tor config v3 --- torData/tor_meek/.gitkeep | 0 torData/tor_none/.gitkeep | 0 torData/tor_obfs4_0/.gitkeep | 0 torData/tor_obfs4_1/.gitkeep | 0 torData/tor_obfs4_2/.gitkeep | 0 torData/tor_snowflake/.gitkeep | 0 torData/tor_webtunnel/.gitkeep | 0 7 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 torData/tor_meek/.gitkeep create mode 100644 torData/tor_none/.gitkeep create mode 100644 torData/tor_obfs4_0/.gitkeep create mode 100644 torData/tor_obfs4_1/.gitkeep create mode 100644 torData/tor_obfs4_2/.gitkeep create mode 100644 torData/tor_snowflake/.gitkeep create mode 100644 torData/tor_webtunnel/.gitkeep diff --git a/torData/tor_meek/.gitkeep b/torData/tor_meek/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_none/.gitkeep b/torData/tor_none/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_obfs4_0/.gitkeep b/torData/tor_obfs4_0/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_obfs4_1/.gitkeep b/torData/tor_obfs4_1/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_obfs4_2/.gitkeep b/torData/tor_obfs4_2/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_snowflake/.gitkeep b/torData/tor_snowflake/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/torData/tor_webtunnel/.gitkeep b/torData/tor_webtunnel/.gitkeep new file mode 100644 index 0000000..e69de29