diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index a37b807..da27252 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -7,25 +7,55 @@ on: workflow_dispatch: # 允许手动触发 jobs: + # 获取版本号 + get-version: + runs-on: ubuntu-20.04 + outputs: + version: ${{ steps.get_version.outputs.version }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version + id: get_version + run: | + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + else + VERSION=$(git describe --tags | grep -o '^v[0-9]\+\.[0-9]\+\.[0-9]\+' | sed 's/^v//' || echo '0.0.0') + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + # 二进制文件构建任务 - build-binaries: + build-and-publish: + needs: get-version strategy: + fail-fast: false matrix: include: - - os: ubuntu-latest - platform: linux + - os: ubuntu-20.04 + platform: ubuntu + arch: x86_64 + - os: ubuntu-22.04 + platform: centos arch: x86_64 - ext: "" + is_centos7: true - os: macos-latest platform: darwin arch: x86_64 - ext: "" + - os: windows-latest + platform: windows + arch: x86_64 runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 @@ -33,135 +63,81 @@ jobs: python-version: '3.11' architecture: x64 - - name: Extract version from tag - id: get_version + - name: Install make (Windows) + if: runner.os == 'Windows' + run: | + choco install make + + - name: Build binary shell: bash + env: + PYTHONPATH: ${{ github.workspace }} + VERSION: ${{ needs.get-version.outputs.version }} + PLATFORM: ${{ matrix.platform }} + ARCH: ${{ matrix.arch }} run: | - if [[ $GITHUB_REF == refs/tags/* ]]; then - echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV + if [ "${{ matrix.is_centos7 }}" = "true" ]; then + make centos7 VERSION=$VERSION else - echo "VERSION=latest" >> $GITHUB_ENV + make clean setup current VERSION=$VERSION fi - - name: Install dependencies + - name: Generate checksum (Linux) + if: runner.os == 'Linux' && !matrix.is_centos7 + shell: bash run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pyinstaller - - - name: Create config directory (Unix) - run: mkdir -p ~/.tapflow + cd dist + sha256sum * > SHA256SUMS.${{ matrix.platform }}.txt - - name: Build binary + - name: Generate checksum (macOS) + if: runner.os == 'macOS' + shell: bash run: | - python -m PyInstaller \ - --clean \ - --name tapflow-${{ env.VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} \ - --add-data "requirements.txt:." \ - --add-data "README.md:." \ - --add-data "tapflow/cli/cli.py:tapflow/cli" \ - --add-data "etc:etc" \ - --add-data "tapflow:tapflow" \ - --hidden-import tapflow \ - --hidden-import tapflow.cli \ - --hidden-import tapflow.cli.cli \ - --hidden-import tapflow.cli.tap \ - --hidden-import tapflow.lib \ - --hidden-import tapflow.lib.configuration \ - --hidden-import tapflow.lib.configuration.config \ - --hidden-import tapflow.lib.backend_apis \ - --hidden-import tapflow.lib.data_pipeline \ - --hidden-import tapflow.lib.connections \ - --hidden-import tapflow.lib.utils \ - --hidden-import tapflow.lib.params \ - --hidden-import tapflow.lib.data_pipeline.validation \ - --hidden-import tapflow.lib.data_services \ - --hidden-import tapflow.lib.system \ - --hidden-import tapflow.lib.cache \ - --hidden-import tapflow.lib.backend_apis.common \ - --hidden-import tapflow.lib.backend_apis.connections \ - --hidden-import tapflow.lib.backend_apis.task \ - --hidden-import tapflow.lib.backend_apis.dataVerify \ - --hidden-import tapflow.lib.backend_apis.metadataInstance \ - --hidden-import tapflow.lib.backend_apis.apiServers \ - --hidden-import IPython \ - --hidden-import yaml \ - --hidden-import requests \ - --hidden-import websockets \ - --hidden-import bson \ - --hidden-import urllib \ - --hidden-import traitlets \ - --hidden-import importlib.metadata \ - --hidden-import logging \ - --hidden-import email \ - --hidden-import xml \ - --hidden-import http \ - --hidden-import ctypes \ - --hidden-import multiprocessing \ - --hidden-import dateutil \ - --hidden-import json \ - --hidden-import asyncio \ - --hidden-import concurrent.futures \ - --hidden-import idna \ - --hidden-import urllib3 \ - --hidden-import charset_normalizer \ - --log-level ERROR \ - --onefile \ - tapflow/cli/tap.py - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: binary-${{ matrix.platform }}-${{ matrix.arch }} - path: dist/*${{ matrix.ext }} - retention-days: 1 + cd dist + for file in *; do + if [ -f "$file" ] && [ "$file" != "SHA256SUMS.darwin.txt" ]; then + shasum -a 256 "$file" >> SHA256SUMS.darwin.txt + fi + done - # 发布二进制文件到服务器任务 - publish-binaries: - needs: [build-binaries] - runs-on: ubuntu-latest - - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: dist - merge-multiple: true + - name: Generate checksum (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cd dist + Get-ChildItem -File | ForEach-Object { + $hash = (Get-FileHash -Algorithm SHA256 $_.Name).Hash.ToLower() + "$hash $($_.Name)" | Out-File -Append -Encoding utf8 SHA256SUMS.windows.txt + } - - name: Install rsync - run: sudo apt-get install -y rsync + - name: Fix line endings (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cd dist + (Get-Content SHA256SUMS.windows.txt) | Set-Content -Encoding utf8 SHA256SUMS.windows.txt - - name: Get version from tag - id: get_version + - name: Install rsync (Windows) + if: runner.os == 'Windows' + shell: pwsh run: | - if [[ $GITHUB_REF == refs/tags/* ]]; then - echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - else - echo "VERSION=latest" >> $GITHUB_OUTPUT - fi + choco install rsync - - name: Prepare artifacts for upload + - name: Deploy to rsync server (Unix) + if: runner.os != 'Windows' + shell: bash + env: + VERSION: ${{ needs.get-version.outputs.version }} run: | - mkdir -p upload - # 显示当前目录结构以便调试 - echo "Current directory structure:" - ls -R dist/ - # 直接复制 dist 目录下的所有文件 - cp -r dist/*/* upload/ || cp -r dist/* upload/ - cd upload - # 确保目录不为空再生成校验和 - if [ "$(ls -A)" ]; then - sha256sum * > SHA256SUMS.txt - else - echo "Error: No files found to generate checksums" - exit 1 - fi + echo '${{ secrets.RSYNC_PASSWORD }}' > /tmp/rsync.passwd && chmod 600 /tmp/rsync.passwd + rsync --password-file=/tmp/rsync.passwd -vzrt --progress dist/ "rsync://${{ secrets.RSYNC_USER }}@${{ secrets.RSYNC_HOST }}/data/enterprise-artifact/gz/tapflow/$VERSION/" || true - - name: Upload to server via rsync + - name: Deploy to rsync server (Windows) + if: runner.os == 'Windows' + shell: pwsh env: + VERSION: ${{ needs.get-version.outputs.version }} RSYNC_PASSWORD: ${{ secrets.RSYNC_PASSWORD }} - RSYNC_HOST: ${{ secrets.RSYNC_HOST }} - VERSION: ${{ steps.get_version.outputs.VERSION }} run: | - echo "${{ env.RSYNC_PASSWORD }}" > /tmp/rsync_password && chmod 600 /tmp/rsync_password - rsync --password-file=/tmp/rsync_password -avz --progress upload/ "rsync://${RSYNC_HOST}/data/enterprise-artifact/gz/tapflow/${VERSION}/" \ No newline at end of file + $env:RSYNC_PASSWORD | Out-File -FilePath "$env:TEMP\rsync.passwd" -Encoding ASCII + rsync --password-file="$env:TEMP\rsync.passwd" -vzrt --progress dist/ "rsync://${{ secrets.RSYNC_USER }}@${{ secrets.RSYNC_HOST }}/data/enterprise-artifact/gz/tapflow/${{ env.VERSION }}/" || exit 0 \ No newline at end of file diff --git a/Makefile b/Makefile index 19b4c8d..35d6d46 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,40 @@ # 定义变量 PACKAGE_NAME = tapflow -VERSION = 0.2.54 +VERSION ?= 0.0.0 MAIN_ENTRY = tapflow/cli/tap.py DIST_DIR = dist BUILD_DIR = build -CONFIG_DIR = $(HOME)/.tapflow +DOCKER_CENTOS7_CMD = docker run --rm -v $(shell pwd):/workspace -w /workspace centos:7 /bin/bash -c + +# 检测操作系统 +ifeq ($(OS),Windows_NT) + PLATFORM = windows + ifeq ($(PROCESSOR_ARCHITECTURE),AMD64) + ARCH = x86_64 + else ifeq ($(PROCESSOR_ARCHITECTURE),x86) + ARCH = x86 + else + ARCH = $(PROCESSOR_ARCHITECTURE) + endif + PYTHON = python + PIP = pip + CONFIG_DIR = $(subst \,/,$(USERPROFILE))/.tapflow + MKDIR = powershell -Command "New-Item -ItemType Directory -Force -Path" + RMRF = powershell -Command "if (Test-Path '$(1)') { Remove-Item -Recurse -Force '$(1)' }" + COPY = powershell -Command "Copy-Item" +else + PLATFORM ?= $(shell uname -s | tr '[:upper:]' '[:lower:]') + ARCH ?= $(shell uname -m) + PYTHON = python3 + PIP = pip3 + CONFIG_DIR = $(HOME)/.tapflow + MKDIR = mkdir -p + RMRF = rm -rf + COPY = cp +endif # Python 虚拟环境 VENV = .venv -PYTHON = $(VENV)/bin/python -PIP = $(VENV)/bin/pip # 默认目标 .PHONY: all @@ -18,7 +43,7 @@ all: clean setup current # 设置虚拟环境 .PHONY: setup setup: - python3 -m venv $(VENV) + $(PYTHON) -m venv $(VENV) $(PIP) install --upgrade pip $(PIP) install -r requirements.txt $(PIP) install pyinstaller @@ -26,25 +51,34 @@ setup: # 清理构建文件 .PHONY: clean clean: - rm -rf $(DIST_DIR) $(BUILD_DIR) - rm -rf *.spec +ifeq ($(OS),Windows_NT) + $(call RMRF,$(DIST_DIR)) + $(call RMRF,$(BUILD_DIR)) + $(call RMRF,*.spec) +else + $(RMRF) "$(DIST_DIR)" "$(BUILD_DIR)" *.spec +endif # 初始化配置文件 .PHONY: init-config init-config: - mkdir -p $(CONFIG_DIR) + $(MKDIR) "$(CONFIG_DIR)" +ifeq ($(OS),Windows_NT) + powershell -Command "if (-not (Test-Path '$(CONFIG_DIR)/config.ini')) { @('[backend]', 'server = localhost', 'access_code = ') | Set-Content '$(CONFIG_DIR)/config.ini' -Encoding UTF8 }" +else @if [ ! -f "$(CONFIG_DIR)/config.ini" ]; then \ - echo "[backend]" > $(CONFIG_DIR)/config.ini; \ - echo "server = localhost" >> $(CONFIG_DIR)/config.ini; \ - echo "access_code = " >> $(CONFIG_DIR)/config.ini; \ + echo "[backend]" > "$(CONFIG_DIR)/config.ini"; \ + echo "server = localhost" >> "$(CONFIG_DIR)/config.ini"; \ + echo "access_code = " >> "$(CONFIG_DIR)/config.ini"; \ fi +endif # 构建当前平台版本 .PHONY: current current: init-config - PYTHONPATH=$(PWD) $(PYTHON) -m PyInstaller \ + $(PYTHON) -m PyInstaller \ --clean \ - --name $(PACKAGE_NAME)-$(VERSION)-$(shell uname -s | tr '[:upper:]' '[:lower:]')-$(shell uname -m) \ + --name $(PACKAGE_NAME)-$(VERSION)-$(PLATFORM)-$(ARCH) \ --add-data "requirements.txt:." \ --add-data "README.md:." \ --add-data "tapflow/cli/cli.py:tapflow/cli" \ @@ -60,11 +94,66 @@ current: init-config --hidden-import tapflow.lib.backend_apis \ --hidden-import tapflow.lib.data_pipeline \ --hidden-import tapflow.lib.connections \ + --hidden-import tapflow.lib.utils \ + --hidden-import tapflow.lib.params \ + --hidden-import tapflow.lib.data_pipeline.validation \ + --hidden-import tapflow.lib.data_services \ + --hidden-import tapflow.lib.system \ + --hidden-import tapflow.lib.cache \ + --hidden-import tapflow.lib.backend_apis.common \ + --hidden-import tapflow.lib.backend_apis.connections \ + --hidden-import tapflow.lib.backend_apis.task \ + --hidden-import tapflow.lib.backend_apis.dataVerify \ + --hidden-import tapflow.lib.backend_apis.metadataInstance \ + --hidden-import tapflow.lib.backend_apis.apiServers \ --hidden-import IPython \ --hidden-import yaml \ --hidden-import requests \ --hidden-import websockets \ --hidden-import bson \ + --hidden-import urllib \ + --hidden-import traitlets \ + --hidden-import importlib.metadata \ + --hidden-import logging \ + --hidden-import email \ + --hidden-import xml \ + --hidden-import http \ + --hidden-import ctypes \ + --hidden-import multiprocessing \ + --hidden-import dateutil \ + --hidden-import json \ + --hidden-import asyncio \ + --hidden-import concurrent.futures \ + --hidden-import idna \ + --hidden-import urllib3 \ + --hidden-import charset_normalizer \ --log-level ERROR \ --onefile \ - $(MAIN_ENTRY) \ No newline at end of file + $(MAIN_ENTRY) + +# CentOS 7构建目标 +.PHONY: centos7 +centos7: + $(DOCKER_CENTOS7_CMD) '\ + curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo && \ + yum makecache && \ + yum install -y make gcc openssl-devel bzip2-devel libffi-devel wget sqlite-devel && \ + wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz && \ + tar xzf Python-3.8.12.tgz && \ + cd Python-3.8.12 && \ + ./configure --enable-optimizations --enable-loadable-sqlite-extensions --enable-shared && \ + make altinstall && \ + cd .. && \ + rm -rf Python-3.8.12* && \ + echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf && \ + ldconfig && \ + ln -sf /usr/local/bin/python3.8 /usr/local/bin/python3 && \ + ln -sf /usr/local/bin/pip3.8 /usr/local/bin/pip3 && \ + python3 -m pip install --upgrade pip && \ + python3 -m pip install "urllib3<2.0.0" && \ + python3 -m pip install importlib-metadata pyinstaller==4.10 && \ + PLATFORM=centos ARCH=x86_64 PYTHON=python3.8 make clean setup current VERSION=$(VERSION) && \ + cd dist && \ + sha256sum * > SHA256SUMS.centos.txt && \ + cd .. && \ + chown -R $(shell id -u):$(shell id -g) dist' \ No newline at end of file