diff --git a/.changeset/config.json b/.changeset/config.json index 17e76724a..8e47c6f9b 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -2,12 +2,19 @@ "$schema": "https://unpkg.com/@changesets/config@3.0.5/schema.json", "changelog": "@changesets/cli/changelog", "commit": false, - "fixed": [], + "fixed": [ + [ + "@ui-tars/*", + "!@ui-tars/visualizer" + ] + ], "linked": [], "access": "restricted", "baseBranch": "main", "updateInternalDependencies": "patch", "ignore": [ - "ui-tars-desktop" + "ui-tars-desktop", + "ui-tars-desktop-renderer", + "@common/*" ] } diff --git a/.changeset/dull-lions-tie.md b/.changeset/dull-lions-tie.md new file mode 100644 index 000000000..fa40913d0 --- /dev/null +++ b/.changeset/dull-lions-tie.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/action-parser': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +chore: same version diff --git a/.changeset/fast-insects-flash.md b/.changeset/fast-insects-flash.md new file mode 100644 index 000000000..8103af884 --- /dev/null +++ b/.changeset/fast-insects-flash.md @@ -0,0 +1,9 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/shared': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +chore: open-operator diff --git a/.changeset/forty-toys-watch.md b/.changeset/forty-toys-watch.md new file mode 100644 index 000000000..101f39fcc --- /dev/null +++ b/.changeset/forty-toys-watch.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/action-parser': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +feat(sdk): action parser add start_coords, end_coords diff --git a/.changeset/four-hairs-dance.md b/.changeset/four-hairs-dance.md new file mode 100644 index 000000000..7dbd07471 --- /dev/null +++ b/.changeset/four-hairs-dance.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/shared': patch +'@ui-tars/sdk': patch +'@ui-tars/action-parser': patch +'@ui-tars/cli': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/utio': patch +--- + +refactor: operator no need return width and height diff --git a/.changeset/hip-pumas-check.md b/.changeset/hip-pumas-check.md new file mode 100644 index 000000000..ad102aad2 --- /dev/null +++ b/.changeset/hip-pumas-check.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/action-parser': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +chore: screenshot bug diff --git a/.changeset/lemon-swans-chew.md b/.changeset/lemon-swans-chew.md new file mode 100644 index 000000000..a1ed5e26f --- /dev/null +++ b/.changeset/lemon-swans-chew.md @@ -0,0 +1,5 @@ +--- +'@ui-tars/sdk': patch +--- + +chore: factors diff --git a/.changeset/modern-zebras-guess.md b/.changeset/modern-zebras-guess.md new file mode 100644 index 000000000..a1ed5e26f --- /dev/null +++ b/.changeset/modern-zebras-guess.md @@ -0,0 +1,5 @@ +--- +'@ui-tars/sdk': patch +--- + +chore: factors diff --git a/.changeset/pre.json b/.changeset/pre.json new file mode 100644 index 000000000..dd14b32ff --- /dev/null +++ b/.changeset/pre.json @@ -0,0 +1,34 @@ +{ + "mode": "exit", + "tag": "beta", + "initialVersions": { + "ui-tars-desktop": "0.0.7-beta.2", + "ui-tars-desktop-renderer": "0.0.1", + "@common/configs": "0.0.1", + "@common/electron-build": "1.0.0", + "@ui-tars/action-parser": "1.2.0-beta.21", + "@ui-tars/cli": "1.2.0-beta.21", + "@ui-tars/electron-ipc": "1.2.0-beta.21", + "@ui-tars/operator-browserbase": "1.2.0-beta.21", + "@ui-tars/operator-nut-js": "1.2.0-beta.21", + "@ui-tars/sdk": "1.2.0-beta.21", + "@ui-tars/shared": "1.2.0-beta.21", + "@ui-tars/utio": "1.2.0-beta.21" + }, + "changesets": [ + "dull-lions-tie", + "fast-insects-flash", + "forty-toys-watch", + "four-hairs-dance", + "hip-pumas-check", + "lemon-swans-chew", + "modern-zebras-guess", + "pretty-brooms-brush", + "selfish-cups-divide", + "selfish-humans-drive", + "short-shoes-tap", + "stale-icons-whisper", + "strange-schools-help", + "witty-points-rescue" + ] +} diff --git a/.changeset/pretty-brooms-brush.md b/.changeset/pretty-brooms-brush.md new file mode 100644 index 000000000..e47cf7177 --- /dev/null +++ b/.changeset/pretty-brooms-brush.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/action-parser': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +fix: test diff --git a/.changeset/selfish-cups-divide.md b/.changeset/selfish-cups-divide.md new file mode 100644 index 000000000..d5d6f0f10 --- /dev/null +++ b/.changeset/selfish-cups-divide.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +'@ui-tars/action-parser': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +--- + +feat: sdk factors version diff --git a/.changeset/selfish-humans-drive.md b/.changeset/selfish-humans-drive.md new file mode 100644 index 000000000..8471c88cb --- /dev/null +++ b/.changeset/selfish-humans-drive.md @@ -0,0 +1,11 @@ +--- +'@ui-tars/action-parser': patch +'@ui-tars/cli': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/sdk': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +--- + +bump: sdk support diff --git a/.changeset/short-shoes-tap.md b/.changeset/short-shoes-tap.md new file mode 100644 index 000000000..438a7b944 --- /dev/null +++ b/.changeset/short-shoes-tap.md @@ -0,0 +1,7 @@ +--- +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/sdk': patch +--- + +chore: types diff --git a/.changeset/stale-icons-whisper.md b/.changeset/stale-icons-whisper.md new file mode 100644 index 000000000..ed396c05d --- /dev/null +++ b/.changeset/stale-icons-whisper.md @@ -0,0 +1,12 @@ +--- +'@ui-tars/action-parser': patch +'@ui-tars/cli': patch +'@ui-tars/electron-ipc': patch +'@ui-tars/operator-browserbase': patch +'@ui-tars/operator-nut-js': patch +'@ui-tars/sdk': patch +'@ui-tars/shared': patch +'@ui-tars/utio': patch +--- + +chore: changeset diff --git a/.changeset/strange-schools-help.md b/.changeset/strange-schools-help.md new file mode 100644 index 000000000..a37e637f2 --- /dev/null +++ b/.changeset/strange-schools-help.md @@ -0,0 +1,5 @@ +--- +'@ui-tars/cli': patch +--- + +chore: node-fetch diff --git a/.changeset/witty-points-rescue.md b/.changeset/witty-points-rescue.md new file mode 100644 index 000000000..30c4c9a12 --- /dev/null +++ b/.changeset/witty-points-rescue.md @@ -0,0 +1,6 @@ +--- +'@ui-tars/cli': patch +'@ui-tars/sdk': patch +--- + +update diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 94de04d9e..f11894d33 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,4 +44,4 @@ jobs: - name: Install dependencies run: pnpm install - name: Run e2e - run: pnpm run test:e2e + run: cd apps/ui-tars && pnpm run test:e2e diff --git a/.github/workflows/release-agent-tars.yml b/.github/workflows/release-agent-tars.yml new file mode 100644 index 000000000..a0d891ec3 --- /dev/null +++ b/.github/workflows/release-agent-tars.yml @@ -0,0 +1,101 @@ +name: Release Agent-TARS app +on: + # push: + # branches: + # - main + workflow_dispatch: + +env: + CI: true + NODE_OPTIONS: --max-old-space-size=8192 + HUSKY: 0 + +permissions: + id-token: write + contents: write + attestations: write + +jobs: + publish_on_mac: + strategy: + fail-fast: false + matrix: + os: [macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Show operating system info + run: | + echo "Operating System:" + uname -a + - name: Install pnpm + run: npm install -g pnpm@9 + - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + with: + node-version: 20 + cache: 'pnpm' + - name: Install Python setuptools + run: brew install python-setuptools + - name: Install appdmg + run: npm install -g appdmg + - name: Install the Apple certificate and provisioning profile + env: + BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }} + P12_PASSWORD: ${{ secrets.P12_PASSWORD }} + BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + run: | + # create variables + CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12 + PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision + KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db + + # import certificate and provisioning profile from secrets + echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH + echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH + + # create temporary keychain + security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH + security set-keychain-settings -lut 21600 $KEYCHAIN_PATH + security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH + + # import certificate to keychain + security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH + security set-key-partition-list -S apple-tool:,apple: -k "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH + security list-keychain -d user -s $KEYCHAIN_PATH + + # apply provisioning profile + mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles + cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles + - name: install dependencies + run: pnpm install + - name: publish arm64 + env: + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + KEYCHAIN_PATH: ${{ runner.temp }}/app-signing.keychain-db + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd apps/omega && pnpm run publish:mac + + # publish_on_win: + # runs-on: windows-latest + # steps: + # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # - name: Show operating system info + # run: | + # echo "Operating System:" + # cmd /c ver + # - name: Install pnpm + # run: npm install -g pnpm@9 + # - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + # with: + # node-version: 20 + # cache: 'pnpm' + # - name: install dependencies + # run: pnpm install + # - name: publish + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # run: cd apps/omega && pnpm run publish:win32 diff --git a/.github/workflows/release.yml b/.github/workflows/release-ui-tars.yml similarity index 95% rename from .github/workflows/release.yml rename to .github/workflows/release-ui-tars.yml index 1cbec951e..c6a0a0184 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-ui-tars.yml @@ -1,4 +1,4 @@ -name: Release app +name: Release UI-TARS app on: # push: # branches: @@ -78,7 +78,7 @@ jobs: KEYCHAIN_PATH: ${{ runner.temp }}/app-signing.keychain-db GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - pnpm run publish:mac-arm64 + cd apps/ui-tars && pnpm run publish:mac-arm64 - name: publish x64 if: matrix.os == 'macos-13' env: @@ -88,7 +88,7 @@ jobs: KEYCHAIN_PATH: ${{ runner.temp }}/app-signing.keychain-db GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - pnpm run publish:mac-x64 + cd apps/ui-tars && pnpm run publish:mac-x64 publish_on_win: runs-on: windows-latest @@ -109,4 +109,4 @@ jobs: - name: publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: pnpm run publish:win32 + run: cd apps/ui-tars && pnpm run publish:win32 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5aed58132..04e0dc712 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,12 +32,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 with: results_file: results.sarif results_format: sarif diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 000000000..dd9d92680 --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,22 @@ +on: + push: + branches: + - main + + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Secret Scanning + uses: trufflesecurity/trufflehog@7dc056a193116ba8d82154bf0549381c8fb8545c # v3.88.14 + with: + extra_args: --results=verified,unknown diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a6a23c48b..3bf6d0141 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: - name: Install dependencies run: pnpm install - name: Run typecheck - run: pnpm run typecheck + run: cd apps/ui-tars && pnpm run typecheck - name: Run test run: pnpm run coverage - name: Upload coverage data @@ -48,19 +48,3 @@ jobs: files: ./coverage/lcov.info token: ${{ secrets.CODECOV_TOKEN }} verbose: false - benchmark: - name: Benchmark - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install pnpm - run: npm install -g pnpm@9 - - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 - with: - node-version: 20 - cache: 'pnpm' - - name: Install dependencies - run: pnpm install - - name: Run benchmark - run: pnpm run test:bench diff --git a/.gitignore b/.gitignore index f47f3bf33..5f4c849fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules dist out +.tmp .DS_Store *.log* .eslintcache diff --git a/.prettierrc.mjs b/.prettierrc.mjs index 14c2c185d..9579ff6ab 100644 --- a/.prettierrc.mjs +++ b/.prettierrc.mjs @@ -22,18 +22,18 @@ export default { useTabs: false, embeddedLanguageFormatting: 'auto', endOfLine: 'auto', - importOrder: [ - '^node:(.*)$', - '', - '^@ui-tars/(.*)$', - '^@main/(.*)$', - '^@shared/(.*)$', - '^@renderer/(.*)$', - '^@resources/(.*)$', - '^[./]', - ], - importOrderSeparation: true, - importOrderSortSpecifiers: true, + // importOrder: [ + // '^node:(.*)$', + // '', + // '^@ui-tars/(.*)$', + // '^@main/(.*)$', + // '^@shared/(.*)$', + // '^@renderer/(.*)$', + // '^@resources/(.*)$', + // '^[./]', + // ], + // importOrderSeparation: true, + // importOrderSortSpecifiers: true, // https://github.com/trivago/prettier-plugin-sort-imports/issues/229 // plugins: ['@trivago/prettier-plugin-sort-imports'], }; diff --git a/README.md b/README.md index 98b2f03e6..4cd1390a7 100644 --- a/README.md +++ b/README.md @@ -1,169 +1 @@ - -

- UI-TARS -

- -# UI-TARS Desktop - -UI-TARS Desktop is a GUI Agent application based on [UI-TARS (Vision-Language Model)](https://github.com/bytedance/UI-TARS) that allows you to control your computer using natural language. - - -

-    📑 Paper    - | 🤗 Hugging Face Models   - |   🤖 ModelScope   -
-🖥️ Desktop Application    -|    👓 Midscene (use in browser) -

- -### ⚠️ Important Announcement: GGUF Model Performance - -The **GGUF model** has undergone quantization, but unfortunately, its performance cannot be guaranteed. As a result, we have decided to **downgrade** it. - -💡 **Alternative Solution**: -You can use **[Cloud Deployment](#cloud-deployment)** or **[Local Deployment [vLLM]](#local-deployment-vllm)**(If you have enough GPU resources) instead. - -We appreciate your understanding and patience as we work to ensure the best possible experience. - -## Updates - -- 🚀 01.25: We updated the **[Cloud Deployment](#cloud-deployment)** section in the 中文版: [GUI模型部署教程](https://bytedance.sg.larkoffice.com/docx/TCcudYwyIox5vyxiSDLlgIsTgWf#U94rdCxzBoJMLex38NPlHL21gNb) with new information related to the ModelScope platform. You can now use the ModelScope platform for deployment. - -## Showcases - -| Instruction | Video | -| :---: | :---: | -| Get the current weather in SF using the web browser |