From d26ca1aac9141d16f55b6aa44adc4c5f88779629 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Mon, 16 Sep 2024 23:48:17 +0300 Subject: [PATCH 01/17] feat(parser): replace with new srcinfo parser --- server/consts/consts.go | 2 +- .../clipboard-bin/clipboard-bin.pacscript | 35 ---- .../clipboard-bin/clipboard-bin.snapshot.json | 35 ---- .../clipboard-with-comments-bin.pacscript | 38 ---- .../clipboard-with-comments-bin.snapshot.json | 39 ---- .../font-downloader/font-downloader.pacscript | 21 --- .../font-downloader.snapshot.json | 37 ---- .../rhino-setup-git/rhino-setup-git.pacscript | 40 ---- .../rhino-setup-git.snapshot.json | 44 ----- .../sample-valid-deb.pacscript | 20 -- .../sample-valid-deb.snapshot.json | 57 ------ ...ample-valid-with-pkgver-func-deb.pacscript | 23 --- ...e-valid-with-pkgver-func-deb.snapshot.json | 57 ------ server/go.mod | 5 +- server/go.sum | 2 + server/server/api/pacscripts/dependencies.go | 20 +- server/server/api/repology/types.go | 6 +- server/server/ssr/pacscript/package.go | 2 +- server/types/equals.go | 5 + server/types/pac/parser/last_updated.go | 7 +- server/types/pac/parser/pacscript.go | 54 +----- .../pac/parser/pacsh/parse_pac_output.go | 173 +----------------- server/types/pac/parser/pacsh/pretty-name.go | 2 +- .../types/pac/parser/pacsh/temp_dir_test.go | 155 ---------------- server/types/pac/parser/parse.go | 33 ++-- server/types/pac/parser/parse_test.go | 32 +++- server/types/pac/parser/search.go | 6 +- server/types/pac/script.go | 103 ++++++++--- 28 files changed, 157 insertions(+), 896 deletions(-) delete mode 100644 server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.pacscript delete mode 100644 server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.snapshot.json delete mode 100644 server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.pacscript delete mode 100644 server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.snapshot.json delete mode 100644 server/fixtures/test-programs/packages/font-downloader/font-downloader.pacscript delete mode 100644 server/fixtures/test-programs/packages/font-downloader/font-downloader.snapshot.json delete mode 100644 server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.pacscript delete mode 100644 server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.snapshot.json delete mode 100644 server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.pacscript delete mode 100644 server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.snapshot.json delete mode 100644 server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.pacscript delete mode 100644 server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.snapshot.json create mode 100644 server/types/equals.go delete mode 100644 server/types/pac/parser/pacsh/temp_dir_test.go diff --git a/server/consts/consts.go b/server/consts/consts.go index fa4a9d40..486d06fd 100644 --- a/server/consts/consts.go +++ b/server/consts/consts.go @@ -1,3 +1,3 @@ package consts -const PACSCRIPT_FILE_EXTENSION = "pacscript" +const SRCINFO_FILE_EXTENSION = ".SRCINFO" diff --git a/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.pacscript b/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.pacscript deleted file mode 100644 index 74d35f2b..00000000 --- a/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.pacscript +++ /dev/null @@ -1,35 +0,0 @@ -# __ __________ ______ -# / \ / \_____ \ / __ \ -# \ \/\/ // ____/ > < -# \ // \/ -- \ -# \__/\ / \_______ \______ / -# \/ \/ \/ -maintainer=("wizard-28 ") - -pkgname="clipboard-bin" -gives="clipboard" -pkgver="0.9.0.1" -pkgdesc="Cut, copy, and paste anything in your terminal" -gives="${gives}" -conflicts=("${gives}" "${gives}-git" "${gives}-deb" "${gives}-app") -arch=("amd64" "arm64" "armhf" "ppc64el" "riscv64") -repology=("project: clipboard") -source=("https://github.com/Slackadays/Clipboard/releases/download/${pkgver}/${gives}-linux-${CARCH}.zip") -sha256sums_amd64=("5b90cd7299c1c0d679cfe8c1bd4e89e7fd70ebede2890d90a6f1da98a90e922b") -sha256sums_arm64=("07493b5e9954585160fc54314e23e4897652f06594f6ec7ceba66b32d7f72b82") -sha256sums_armhf=("7654d6f5176e554ed86d84f16924b2ec3d7a7e0000f24a43ee6772397b986dea") -sha256sums_ppc64el=("a7c2c689a777d57fe6638a469c408753d1b4d5d61c8fecd141a4781f54a24e7a") -sha256sums_riscv64=("e92f2c4eeeefd093d25f91f186c1c3ac572ea254369fe7028928246d431407c8") - - -package() { - cd "${_archive}" - if [[ ${CARCH} == "amd64" ]]; then - sudo install -Dm 755 "lib/libcbwayland.so" "${pkgdir}/usr/lib/libcbwayland.so" - fi - sudo install -Dm 755 "lib/libcbx11.so" "${pkgdir}/usr/lib/libcbx11.so" - - sudo install -Dm 755 "bin/cb" "${pkgdir}/usr/bin/cb" -} - -# vim:set ft=sh ts=2 sw=2 et: diff --git a/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.snapshot.json b/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.snapshot.json deleted file mode 100644 index d5af1e11..00000000 --- a/server/fixtures/test-programs/packages/clipboard-bin/clipboard-bin.snapshot.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "prettyName": "Clipboard", - "version": "0.9.0.1", - "latestVersion": null, - "packageName": "clipboard-bin", - "maintainers": [ - "wizard-28 \u003cwiz28@pm.me\u003e" - ], - "description": "Cut, copy, and paste anything in your terminal", - "source": [ - "https://github.com/Slackadays/Clipboard/releases/download/0.9.0.1/clipboard-linux-amd64.zip" - ], - "runtimeDependencies": [], - "buildDependencies": [], - "optionalDependencies": [], - "conflicts": [ - "clipboard", - "clipboard-git", - "clipboard-deb", - "clipboard-app" - ], - "breaks": [], - "gives": "clipboard", - "replaces": [], - "hash": null, - "ppa": [], - "pacstallDependencies": [], - "patch": [], - "repology": [ - "project: clipboard" - ], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.pacscript b/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.pacscript deleted file mode 100644 index c75b83d7..00000000 --- a/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.pacscript +++ /dev/null @@ -1,38 +0,0 @@ -# __ __________ ______ -# / \ / \_____ \ / __ \ -# \ \/\/ // ____/ > < -# \ // \/ -- \ -# \__/\ / \_______ \______ / -# \/ \/ \/ -maintainer=("wizard-28 ") - -pkgname="clipboard-bin" -gives="clipboard" -pkgver="0.9.0.1" -pkgdesc="Cut, copy, and paste anything in your terminal" -gives="${gives}" -conflicts=("${gives}" "${gives}-git" "${gives}-deb" "${gives}-app") -arch=("amd64" "arm64" "armhf" "ppc64el" "riscv64") -repology=("project: clipboard") -optdepends=("gnome-disk-utility" - "epiphany-browser: Default browser CMD if not set" - "gnome-control-center: Default wifi CMD if not set") -source=("https://github.com/Slackadays/Clipboard/releases/download/${pkgver}/${gives}-linux-${CARCH}.zip") -sha256sums_amd64=("5b90cd7299c1c0d679cfe8c1bd4e89e7fd70ebede2890d90a6f1da98a90e922b") -sha256sums_arm64=("07493b5e9954585160fc54314e23e4897652f06594f6ec7ceba66b32d7f72b82") -sha256sums_armhf=("7654d6f5176e554ed86d84f16924b2ec3d7a7e0000f24a43ee6772397b986dea") -sha256sums_ppc64el=("a7c2c689a777d57fe6638a469c408753d1b4d5d61c8fecd141a4781f54a24e7a") -sha256sums_riscv64=("e92f2c4eeeefd093d25f91f186c1c3ac572ea254369fe7028928246d431407c8") - - -package() { - cd "${_archive}" - if [[ ${CARCH} == "amd64" ]]; then - sudo install -Dm 755 "lib/libcbwayland.so" "${pkgdir}/usr/lib/libcbwayland.so" - fi - sudo install -Dm 755 "lib/libcbx11.so" "${pkgdir}/usr/lib/libcbx11.so" - - sudo install -Dm 755 "bin/cb" "${pkgdir}/usr/bin/cb" -} - -# vim:set ft=sh ts=2 sw=2 et: diff --git a/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.snapshot.json b/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.snapshot.json deleted file mode 100644 index 7e1c2111..00000000 --- a/server/fixtures/test-programs/packages/clipboard-with-comments-bin/clipboard-with-comments-bin.snapshot.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "prettyName": "Clipboard", - "version": "0.9.0.1", - "latestVersion": null, - "packageName": "clipboard-bin", - "maintainers": [ - "wizard-28 \u003cwiz28@pm.me\u003e" - ], - "description": "Cut, copy, and paste anything in your terminal", - "source": [ - "https://github.com/Slackadays/Clipboard/releases/download/0.9.0.1/clipboard-linux-amd64.zip" - ], - "runtimeDependencies": [], - "buildDependencies": [], - "optionalDependencies": [ - "gnome-disk-utility", - "epiphany-browser: Default browser CMD if not set", - "gnome-control-center: Default wifi CMD if not set" - ], - "conflicts": [ - "clipboard", - "clipboard-git", - "clipboard-deb", - "clipboard-app" - ], - "breaks": [], - "gives": "clipboard", - "replaces": [], - "hash": null, - "ppa": [], - "pacstallDependencies": [], - "patch": [], - "repology": [ - "project: clipboard" - ], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/font-downloader/font-downloader.pacscript b/server/fixtures/test-programs/packages/font-downloader/font-downloader.pacscript deleted file mode 100644 index 856042f5..00000000 --- a/server/fixtures/test-programs/packages/font-downloader/font-downloader.pacscript +++ /dev/null @@ -1,21 +0,0 @@ -pkgname="font-downloader" -gives="font-downloader" -pkgver="10.0.0" -makedepends=("meson" "libhandy-1-dev" "gettext") -# HACK: https://github.com/pacstall/pacstall/issues/727 -depends=("python3-gi" "libhandy-1-dev") -source=("https://github.com/GustavoPeredo/Font-Downloader/archive/refs/tags/v${pkgver}.zip") -repology=("project: fontdownloader") -pkgdesc="Install fonts from online sources" -sha256sums=("eeafd4ac9cb0d47fd0c1512e07805d0f7a639cdbbc688647249eaee8d1753e23") -maintainer=("সৌম্যদীপ ") - -build() { - cd "${_archive}" - meson --prefix=/usr build - ninja -C build -} -package() { - cd "${_archive}" - DESTDIR="${pkgdir}" ninja -C build install -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/font-downloader/font-downloader.snapshot.json b/server/fixtures/test-programs/packages/font-downloader/font-downloader.snapshot.json deleted file mode 100644 index dad185e4..00000000 --- a/server/fixtures/test-programs/packages/font-downloader/font-downloader.snapshot.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "prettyName": "Font Downloader", - "version": "10.0.0", - "latestVersion": null, - "packageName": "font-downloader", - "maintainers": [ - "সৌম্যদীপ \u003csoumyadeepghosh2004@zohomail.in\u003e" - ], - "description": "Install fonts from online sources", - "source": [ - "https://github.com/GustavoPeredo/Font-Downloader/archive/refs/tags/v10.0.0.zip" - ], - "runtimeDependencies": [ - "python3-gi", - "libhandy-1-dev" - ], - "buildDependencies": [ - "meson", - "libhandy-1-dev", - "gettext" - ], - "optionalDependencies": [], - "conflicts": [], - "breaks": [], - "gives": "font-downloader", - "replaces": [], - "hash": null, - "ppa": [], - "pacstallDependencies": [], - "patch": [], - "repology": [ - "project: fontdownloader" - ], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.pacscript b/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.pacscript deleted file mode 100644 index fab6124e..00000000 --- a/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.pacscript +++ /dev/null @@ -1,40 +0,0 @@ -pkgname="rhino-setup-git" -source=("https://github.com/rhino-linux/rhino-setup.git") -pkgver="2023.3" -pkgrel="2" -makedepends=("libgtk-4-dev" "libadwaita-1-dev" "gettext" "desktop-file-utils" "rustc" "cargo" "meson" "ninja-build") -depends=("libgtk-4-dev" "libadwaita-1-dev" "gettext" "desktop-file-utils") -gives="rhino-setup" -replaces="${gives}-bin" -pkgdesc="Rhino Linux Setup Prompt" -maintainer=("Oren Klopfer ") -incompatible=("debian:*") - -build() { - cd "${_archive}" - sudo meson build -} - -package() { - cd "${_archive}" - sudo DESTDIR="${pkgdir}" ninja -C build install -} - -post_install() { - for i in "${homedir}" "/etc/skel"; do - if ! [[ -d "${i}/.config/autostart" ]]; then - mkdir -p "${i}/.config/autostart" - fi - if ! [[ -f "${i}/.config/autostart/rhino-setup.desktop" ]]; then - sudo ln -sf "/usr/local/share/applications/org.rhinolinux.RhinoSetup.desktop" "${i}/.config/autostart/rhino-setup.desktop" - fi - done -} - -post_remove() { - for i in "${homedir}" "/etc/skel"; do - if [[ -L "${i}/.config/autostart/rhino-setup.desktop" ]]; then - sudo rm -f "${i}/.config/autostart/rhino-setup.desktop" - fi - done -} diff --git a/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.snapshot.json b/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.snapshot.json deleted file mode 100644 index 0f698d94..00000000 --- a/server/fixtures/test-programs/packages/rhino-setup-git/rhino-setup-git.snapshot.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "prettyName": "Rhino Setup", - "version": "2023.3", - "latestVersion": null, - "packageName": "rhino-setup-git", - "maintainers": [ - "Oren Klopfer \u003coren@taumoda.com\u003e" - ], - "description": "Rhino Linux Setup Prompt", - "source": [ - "https://github.com/rhino-linux/rhino-setup.git" - ], - "runtimeDependencies": [ - "libgtk-4-dev", - "libadwaita-1-dev", - "gettext", - "desktop-file-utils" - ], - "buildDependencies": [ - "libgtk-4-dev", - "libadwaita-1-dev", - "gettext", - "desktop-file-utils", - "rustc", - "cargo", - "meson", - "ninja-build" - ], - "optionalDependencies": [], - "conflicts": [], - "breaks": [], - "gives": "rhino-setup", - "replaces": [ - "rhino-setup-bin" - ], - "hash": null, - "ppa": [], - "pacstallDependencies": [], - "patch": [], - "repology": [], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.pacscript b/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.pacscript deleted file mode 100644 index 6ef871b4..00000000 --- a/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.pacscript +++ /dev/null @@ -1,20 +0,0 @@ -pkgname="sample-valid-deb" -gives="sample-valid" -repology=("project: sample-valid") -pkgver="1.0.0" -source="https://example.com" -pkgdesc="Sample description" -hash="10101010" -arch=('amd64') -makedepends=("go" "gcc") -optdepends=("opt1" "opt2") -pacdeps=("pacdep1" "pacdep2") -depends=("dep1" "dep2") -ppa=("ppa1" "ppa2") -patch=("patch1" "patch2") -provides=("provides1" "provides2") -incompatible=("incompatible1" "incompatible2") -maintainer=("1234 ") -breaks=("breaks1" "breaks2") -conflicts=("conflicts1" "conflicts2") -replaces=("replaces1" "replaces2") diff --git a/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.snapshot.json b/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.snapshot.json deleted file mode 100644 index a660408a..00000000 --- a/server/fixtures/test-programs/packages/sample-valid-deb/sample-valid-deb.snapshot.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "prettyName": "Sample Valid", - "version": "1.0.0", - "latestVersion": null, - "packageName": "sample-valid-deb", - "maintainers": [ - "1234 \u003ctest@pacstall.dev\u003e" - ], - "description": "Sample description", - "source": [ - "https://example.com" - ], - "runtimeDependencies": [ - "dep1", - "dep2" - ], - "buildDependencies": [ - "go", - "gcc" - ], - "optionalDependencies": [ - "opt1", - "opt2" - ], - "conflicts": [ - "conflicts1", - "conflicts2" - ], - "breaks": [ - "breaks1", - "breaks2" - ], - "gives": "sample-valid", - "replaces": [ - "replaces1", - "replaces2" - ], - "hash": "10101010", - "ppa": [ - "ppa1", - "ppa2" - ], - "pacstallDependencies": [ - "pacdep1", - "pacdep2" - ], - "patch": [ - "patch1", - "patch2" - ], - "repology": [ - "project: sample-valid" - ], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.pacscript b/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.pacscript deleted file mode 100644 index b75845f6..00000000 --- a/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.pacscript +++ /dev/null @@ -1,23 +0,0 @@ -pkgname="sample-valid-deb" -gives="sample-valid" -repology=("project: sample-valid") -source=("https://example.com") -pkgdesc="Sample description" -hash="10101010" -arch=('amd64') -makedepends=("go" "gcc") -optdepends=("opt1" "opt2") -pacdeps=("pacdep1" "pacdep2") -depends=("dep1" "dep2") -ppa=("ppa1" "ppa2") -patch=("patch1" "patch2") -provides=("provides1" "provides2") -incompatible=("incompatible1" "incompatible2") -maintainer=("pacstall ") -breaks=("breaks1" "breaks2") -conflicts=("conflicts1" "conflicts2") -replaces=("replaces1" "replaces2") - -pkgver() { - echo "1.2.3" -} \ No newline at end of file diff --git a/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.snapshot.json b/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.snapshot.json deleted file mode 100644 index eb11e970..00000000 --- a/server/fixtures/test-programs/packages/sample-valid-with-pkgver-func-deb/sample-valid-with-pkgver-func-deb.snapshot.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "prettyName": "Sample Valid", - "version": "1.2.3", - "latestVersion": null, - "packageName": "sample-valid-deb", - "maintainers": [ - "pacstall \u003ctest@pacstall.dev\u003e" - ], - "description": "Sample description", - "source": [ - "https://example.com" - ], - "runtimeDependencies": [ - "dep1", - "dep2" - ], - "buildDependencies": [ - "go", - "gcc" - ], - "optionalDependencies": [ - "opt1", - "opt2" - ], - "conflicts": [ - "conflicts1", - "conflicts2" - ], - "breaks": [ - "breaks1", - "breaks2" - ], - "gives": "sample-valid", - "replaces": [ - "replaces1", - "replaces2" - ], - "hash": "10101010", - "ppa": [ - "ppa1", - "ppa2" - ], - "pacstallDependencies": [ - "pacdep1", - "pacdep2" - ], - "patch": [ - "patch1", - "patch2" - ], - "repology": [ - "project: sample-valid" - ], - "requiredBy": [], - "lastUpdatedAt": "0001-01-01T00:00:00Z", - "updateStatus": -1 -} \ No newline at end of file diff --git a/server/go.mod b/server/go.mod index b6a98d74..6b99c7f4 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,6 +1,8 @@ module pacstall.dev/webserver -go 1.18 +go 1.22.2 + +toolchain go1.22.6 require ( github.com/fatih/color v1.17.0 // for colorizing output @@ -29,6 +31,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pacstall/go-srcinfo v0.0.0-20240916164747-6f75fe362117 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/server/go.sum b/server/go.sum index d95fb57e..25891e12 100644 --- a/server/go.sum +++ b/server/go.sum @@ -41,6 +41,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pacstall/go-srcinfo v0.0.0-20240916164747-6f75fe362117 h1:UVCtjWIdoAxLgASvjvuDKcKfRq1bmy5nGOIBinLZ9vE= +github.com/pacstall/go-srcinfo v0.0.0-20240916164747-6f75fe362117/go.mod h1:0LUf6eSIfP1kloShefbkOeYgk8aUbkeDuN955aq1jxY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/server/server/api/pacscripts/dependencies.go b/server/server/api/pacscripts/dependencies.go index 2aa2cc25..07ddc522 100644 --- a/server/server/api/pacscripts/dependencies.go +++ b/server/server/api/pacscripts/dependencies.go @@ -5,7 +5,6 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/log" "pacstall.dev/webserver/server" "pacstall.dev/webserver/types/array" "pacstall.dev/webserver/types/pac" @@ -13,10 +12,10 @@ import ( ) type pacscriptDependencies struct { - RuntimeDependencies []string `json:"runtimeDependencies"` - BuildDependencies []string `json:"buildDependencies"` - OptionalDependencies []string `json:"optionalDependencies"` - PacstallDependencies []*pac.Script `json:"pacstallDependencies"` + RuntimeDependencies []pac.ArchDistroString `json:"runtimeDependencies"` + BuildDependencies []pac.ArchDistroString `json:"buildDependencies"` + OptionalDependencies []pac.ArchDistroString `json:"optionalDependencies"` + PacstallDependencies []pac.ArchDistroString `json:"pacstallDependencies"` } func GetPacscriptDependenciesHandle(w http.ResponseWriter, req *http.Request) { @@ -45,20 +44,11 @@ func GetPacscriptDependenciesHandle(w http.ResponseWriter, req *http.Request) { return } - pacstallDependencies := make([]*pac.Script, 0) - for _, pkg := range pacpkg.PacstallDependencies { - if found, err := array.FindBy(allPacscripts, func(pi *pac.Script) bool { return pkg == pi.PackageName }); err == nil { - pacstallDependencies = append(pacstallDependencies, found) - } else { - log.Error("could not find pacstall dependency %s of package %s.\n", pkg, pacpkg.PackageName) - } - } - response := pacscriptDependencies{ RuntimeDependencies: pacpkg.RuntimeDependencies, BuildDependencies: pacpkg.BuildDependencies, OptionalDependencies: pacpkg.OptionalDependencies, - PacstallDependencies: pacstallDependencies, + PacstallDependencies: pacpkg.PacstallDependencies, } server.Json(w, response) diff --git a/server/server/api/repology/types.go b/server/server/api/repology/types.go index 98d08fe1..eb1600e4 100644 --- a/server/server/api/repology/types.go +++ b/server/server/api/repology/types.go @@ -23,13 +23,12 @@ type repologyPackage struct { RecipeURL string `json:"recipeUrl"` PackageDetailsURL string `json:"packageDetailsUrl"` Type string `json:"type"` - Patches []string `json:"patches"` } func newRepologyPackage(p *pac.Script) repologyPackage { var source *string = nil if len(p.Source) > 0 { - source = &p.Source[0] + source = &p.Source[0].Value } return repologyPackage{ @@ -40,9 +39,8 @@ func newRepologyPackage(p *pac.Script) repologyPackage { Version: p.Version, URL: source, Type: getType(p), - RecipeURL: fmt.Sprintf("https://raw.githubusercontent.com/pacstall/pacstall-programs/master/packages/%s/%s.%s", p.PackageName, p.PackageName, consts.PACSCRIPT_FILE_EXTENSION), + RecipeURL: fmt.Sprintf("https://raw.githubusercontent.com/pacstall/pacstall-programs/master/packages/%s/%s.%s", p.PackageName, p.PackageName, consts.SRCINFO_FILE_EXTENSION), PackageDetailsURL: fmt.Sprintf("https://pacstall.dev/packages/%s", p.PackageName), - Patches: p.Patch, } } diff --git a/server/server/ssr/pacscript/package.go b/server/server/ssr/pacscript/package.go index 57d656f6..4edeafb7 100644 --- a/server/server/ssr/pacscript/package.go +++ b/server/server/ssr/pacscript/package.go @@ -50,7 +50,7 @@ func registerPacscriptSSRData() {

Find similar packages here.

- `, pkg.PackageName, pkg.PackageName, pkg.Description, strings.Join(pkg.Maintainers, ", "), pkg.Version, pkg.PackageName, pkg.PackageName, consts.PACSCRIPT_FILE_EXTENSION, pkg.PackageName), + `, pkg.PackageName, pkg.PackageName, pkg.Description, strings.Join(pkg.Maintainers, ", "), pkg.Version, pkg.PackageName, pkg.PackageName, consts.SRCINFO_FILE_EXTENSION, pkg.PackageName), } }, ) diff --git a/server/types/equals.go b/server/types/equals.go new file mode 100644 index 00000000..a64bf68e --- /dev/null +++ b/server/types/equals.go @@ -0,0 +1,5 @@ +package types + +type Equaller interface { + Equals(other Equaller) bool +} diff --git a/server/types/pac/parser/last_updated.go b/server/types/pac/parser/last_updated.go index dfd9e3b8..adf1bb8d 100644 --- a/server/types/pac/parser/last_updated.go +++ b/server/types/pac/parser/last_updated.go @@ -10,7 +10,6 @@ import ( "github.com/joomcode/errorx" "pacstall.dev/webserver/config" - "pacstall.dev/webserver/consts" "pacstall.dev/webserver/log" "pacstall.dev/webserver/types/array" "pacstall.dev/webserver/types/pac" @@ -31,8 +30,8 @@ func getPackageLastUpdatedTuples() ([]packageLastUpdatedTuple, error) { programsPath := path.Join(wordingDirectoryAbsolute, config.GitClonePath) script := fmt.Sprintf(` cd %v - for i in ./packages/*/*.%s; do echo $i; git log -1 --pretty=\"%%at\" $i; done - `, programsPath, consts.PACSCRIPT_FILE_EXTENSION) + for i in ./packages/*/*.pacscript; do echo $i; git log -1 --pretty=\"%%at\" $i; done + `, programsPath) outputBytes, err := pacsh.ExecBash(programsPath, "last_updated.sh", []byte(script)) if err != nil { @@ -52,7 +51,7 @@ func getPackageLastUpdatedTuples() ([]packageLastUpdatedTuple, error) { lastUpdatedString = lastUpdatedString[1 : len(lastUpdatedString)-1] packageNameWithExtension := path.Base(packagePath) - packageName := strings.TrimSuffix(packageNameWithExtension, "."+consts.PACSCRIPT_FILE_EXTENSION) + packageName := strings.TrimSuffix(packageNameWithExtension, ".pacscript") if packageName == "" || strings.HasPrefix(packageName, "-") { return nil, errorx.IllegalState.New("failed to parse package name from package path '%v'", packagePath) diff --git a/server/types/pac/parser/pacscript.go b/server/types/pac/parser/pacscript.go index 616ff5b0..82c3032b 100644 --- a/server/types/pac/parser/pacscript.go +++ b/server/types/pac/parser/pacscript.go @@ -1,66 +1,18 @@ package parser import ( - "fmt" - "strings" - "pacstall.dev/webserver/types/array" "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh" ) -func removeDebianCheck(script string) string { - if !strings.Contains(script, "/etc/os-release)\" == \"debian\"") { - return script - } - - if strings.Index(script, "if") != 0 { - return script - } - - debianCheckEnd := strings.Index(script, "fi") - if debianCheckEnd == -1 { - return script - } - - return script[debianCheckEnd+len("fi"):] -} - -func buildCustomFormatScript(header []byte) []byte { - // TODO: remove after `preinstall` gets implemented - script := removeDebianCheck(string(header)) + "\n" - - script += "echo ''\n" - for _, bashName := range pacsh.PacscriptVars { - // If the variable is a function, then we replace it with the output of the function - script += fmt.Sprintf(` -if [[ "$(declare -F -p %v)" ]]; then - %v=$(%v) -fi -`, bashName, bashName, bashName) - } - - script = script + "\njo -p -- " - - for _, bashName := range pacsh.PacscriptVars { - script += fmt.Sprintf("-s %v=\"$%v\" ", bashName, bashName) - } - - for _, bashName := range pacsh.PacscriptArrays { - script += fmt.Sprintf("%v=$(jo -a ${%v[@]}) ", bashName, bashName) - } - - return []byte(script) -} - func computeRequiredBy(script *pac.Script, scripts []*pac.Script) { - pickBeforeColon := func(it *array.Iterator[string]) string { - return strings.Split(it.Value, ": ")[0] + pickName := func(it *array.Iterator[pac.ArchDistroString]) string { + return it.Value.Value } script.RequiredBy = make([]string, 0) for _, otherScript := range scripts { - otherScriptDependencies := array.Map(otherScript.PacstallDependencies, pickBeforeColon) + otherScriptDependencies := array.SwitchMap(otherScript.PacstallDependencies, pickName) if array.Contains(otherScriptDependencies, array.Is(script.PackageName)) { script.RequiredBy = append(script.RequiredBy, otherScript.PackageName) } diff --git a/server/types/pac/parser/pacsh/parse_pac_output.go b/server/types/pac/parser/pacsh/parse_pac_output.go index d6d590c8..21fd336f 100644 --- a/server/types/pac/parser/pacsh/parse_pac_output.go +++ b/server/types/pac/parser/pacsh/parse_pac_output.go @@ -1,178 +1,19 @@ package pacsh import ( - "encoding/json" - "strings" - - "github.com/joomcode/errorx" - "pacstall.dev/webserver/types/array" + "github.com/pacstall/go-srcinfo" "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh/internal" ) -var ParsePacOutput = parseOutput -var PacscriptVars []string = []string{"pkgname", "pkgdesc", "gives", "hash", "pkgver"} -var PacscriptArrays []string = []string{"source", "arch", "maintainer", "depends", "conflicts", "breaks", "replaces", "makedepends", "optdepends", "pacdeps", "patch", "ppa", "repology"} - -type Stringable struct { - Data string -} - -func (s *Stringable) UnmarshalJSON(data []byte) error { - s.Data = string(data) - s.Data = strings.ReplaceAll(s.Data, "\"", "") - return nil -} - -func (s *Stringable) String() string { - return s.Data -} - -type StringableArrayWithComments struct { - Data []Stringable -} - -func (s StringableArrayWithComments) toStringArray() []string { - return array.SwitchMapPtr(s.Data, func(it *array.PtrIterator[Stringable]) string { - val := *it.Value - return val.String() - }) -} - -func (s *StringableArrayWithComments) UnmarshalJSON(data []byte) error { - err := json.Unmarshal(data, &s.Data) - if err != nil { - return err - } - - out := make([]string, 0) - item := "" - hasComments := false - for _, word := range s.Data { - strWord := word.String() - wordHasComment := strings.HasSuffix(strWord, ":") - - if wordHasComment { - hasComments = true - if item != "" { - out = append(out, strings.TrimSpace(item)) - } - item = strWord - } else if !hasComments { - out = append(out, strWord) - } else { - item += " " + strWord - } - - } - - if item != "" { - out = append(out, strings.TrimSpace(item)) - } - - s.Data = array.SwitchMap(out, func(it *array.Iterator[string]) Stringable { - return Stringable{it.Value} - }) - - return nil -} - -type pacscriptJsonStructure struct { - Pkgname string `json:"pkgname"` - Pkgdesc string `json:"pkgdesc"` - Gives *string `json:"gives"` - Hash *string `json:"hash"` - Pkgver *string `json:"pkgver"` - Source StringableArrayWithComments `json:"source"` - Maintainer StringableArrayWithComments `json:"maintainer"` - Depends StringableArrayWithComments `json:"depends"` - Conflicts StringableArrayWithComments `json:"conflicts"` - Arch StringableArrayWithComments `json:"arch"` - Breaks StringableArrayWithComments `json:"breaks"` - Replaces StringableArrayWithComments `json:"replaces"` - Makedepends StringableArrayWithComments `json:"makedepends"` - Optdepends StringableArrayWithComments `json:"optdepends"` - Pacdeps StringableArrayWithComments `json:"pacdeps"` - Patch StringableArrayWithComments `json:"patch"` - Ppa StringableArrayWithComments `json:"ppa"` - Repology StringableArrayWithComments `json:"repology"` -} - -var _GIT_VERSION = "git" -var _EMPTI_STR = "" - -func parseOutput(data []byte) (out pac.Script, err error) { - // remove prefixes if any - runeIndex := strings.IndexRune(string(data), '{') - if runeIndex >= 0 { - str := string(data) - str = str[runeIndex:] - data = []byte(str) - } - - var parsedContent pacscriptJsonStructure - err = json.Unmarshal(data, &parsedContent) +func ParsePacOutput(data []byte) (*pac.Script, error) { + out, err := srcinfo.Parse(string(data)) if err != nil { - return out, errorx.IllegalFormat.Wrap(err, "failed to deserialize json content '%v'", string(data)) - } - if parsedContent.Pkgver == nil { - if strings.HasSuffix(parsedContent.Pkgname, "-git") { - parsedContent.Pkgver = &_GIT_VERSION - } else { - return out, errorx.IllegalArgument.New("expected version to be non-empty") - } + return nil, err } - if parsedContent.Gives == nil { - parsedContent.Gives = &_EMPTI_STR - } + ps := pac.FromSrcInfo(*out) - out = pac.Script{ - PackageName: parsedContent.Pkgname, - Maintainers: parseMaintainers(parsedContent.Maintainer.toStringArray()), - Description: parsedContent.Pkgdesc, - Source: parsedContent.Source.toStringArray(), - Gives: *parsedContent.Gives, - Hash: parsedContent.Hash, - Version: *parsedContent.Pkgver, - RuntimeDependencies: parsedContent.Depends.toStringArray(), - BuildDependencies: parsedContent.Makedepends.toStringArray(), - OptionalDependencies: parsedContent.Optdepends.toStringArray(), - Conflicts: parsedContent.Conflicts.toStringArray(), - Replaces: parsedContent.Replaces.toStringArray(), - Breaks: parsedContent.Breaks.toStringArray(), - PacstallDependencies: parsedContent.Pacdeps.toStringArray(), - PPA: parsedContent.Ppa.toStringArray(), - Patch: parsedContent.Patch.toStringArray(), - RequiredBy: make([]string, 0), - Repology: parsedContent.Repology.toStringArray(), - LatestVersion: nil, - UpdateStatus: pac.UpdateStatus.Unknown, - } - - if pkgver, err := internal.NewGitSources(out.Source).ParseGitPackageVersion(); err == nil && pkgver != "" { - out.Version = pkgver - } - - if out.Hash != nil && len(*out.Hash) == 0 { - out.Hash = nil - } - - out.PrettyName = getPrettyName(out) - - return -} - -func parseMaintainers(maintainers []string) []string { - maintainersSplitByLA := strings.Split(strings.Join(maintainers, " "), ">") - - out := []string{} - for _, maintainer := range maintainersSplitByLA { - if len(maintainer) == 0 { - continue - } - out = append(out, strings.TrimSpace(maintainer)+">") - } + ps.PrettyName = getPrettyName(ps) - return out + return ps, nil } diff --git a/server/types/pac/parser/pacsh/pretty-name.go b/server/types/pac/parser/pacsh/pretty-name.go index 1bed2dd1..dc071fdd 100644 --- a/server/types/pac/parser/pacsh/pretty-name.go +++ b/server/types/pac/parser/pacsh/pretty-name.go @@ -13,7 +13,7 @@ var pacTypes = map[string]string{ "-app": "AppImage", } -func getPrettyName(p pac.Script) string { +func getPrettyName(p *pac.Script) string { name := "" if name == "" { diff --git a/server/types/pac/parser/pacsh/temp_dir_test.go b/server/types/pac/parser/pacsh/temp_dir_test.go deleted file mode 100644 index ce00df49..00000000 --- a/server/types/pac/parser/pacsh/temp_dir_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package pacsh - -import ( - "os" - "strings" - "testing" - "time" - - "pacstall.dev/webserver/types/array" -) - -func cleanup() { - statFile = os.Stat - removeAll = os.RemoveAll - makeDir = os.Mkdir - removeFile = os.Remove -} - -type testFileInfo struct { - name string - size int64 - mode os.FileMode - modTime time.Time - isDir bool -} - -func (t testFileInfo) Name() string { - return t.name -} - -func (t testFileInfo) Size() int64 { - return t.size -} - -func (t testFileInfo) Mode() os.FileMode { - return t.mode -} - -func (t testFileInfo) ModTime() time.Time { - return t.modTime -} - -func (t testFileInfo) IsDir() bool { - return t.isDir -} - -func (t testFileInfo) Sys() interface{} { - return nil -} - -func Test_CreateTempDirectory_NoExisting(t *testing.T) { - defer cleanup() - - makeDirCalled := 0 - removeDirCalled := 0 - statFileCalled := 0 - - statFile = func(path string) (os.FileInfo, error) { - if statFileCalled == 1 { - statFileCalled += 1 - return nil, os.ErrNotExist - } - - statFileCalled += 1 - name, _ := array.Last(strings.Split(path, "/")) - return testFileInfo{ - name: name, - size: 0, - mode: 0777, - modTime: time.Now(), - isDir: true, - }, nil - } - - makeDir = func(path string, perm os.FileMode) error { - makeDirCalled += 1 - return nil - } - - removeAll = func(path string) error { - removeDirCalled += 1 - return nil - } - - err := CreateTempDirectory("/tmp") - - if err != nil { - t.Errorf("expected no error, got %v", err) - } - - if removeDirCalled != 1 { - t.Error("expected removeAll to be called 1 time but was called", removeDirCalled) - } - - if makeDirCalled != 1 { - t.Error("expected makeDir to be called 1 time but was called", makeDirCalled) - } - - if statFileCalled != 2 { - t.Error("expected statFile to be called 2 times but was called", statFileCalled) - } -} - -func Test_CreateTempDirectory_AlreadyExisting(t *testing.T) { - defer cleanup() - - makeDirCalled := 0 - removeDirCalled := 0 - statFileCalled := 0 - - statFile = func(path string) (os.FileInfo, error) { - if statFileCalled == 0 { - statFileCalled += 1 - return nil, os.ErrNotExist - } - - statFileCalled += 1 - name, _ := array.Last(strings.Split(path, "/")) - return testFileInfo{ - name: name, - size: 0, - mode: 0777, - modTime: time.Now(), - isDir: true, - }, nil - } - - makeDir = func(path string, perm os.FileMode) error { - makeDirCalled += 1 - return nil - } - - removeAll = func(path string) error { - removeDirCalled += 1 - return nil - } - - err := CreateTempDirectory("/tmp") - - if err != nil { - t.Errorf("expected no error, got %v", err) - } - - if removeDirCalled != 0 { - t.Error("expected removeAll to be called 0 times but was called", removeDirCalled) - } - - if makeDirCalled != 1 { - t.Error("expected makeDir to be called 1 time but was called", makeDirCalled) - } - - if statFileCalled != 1 { - t.Error("expected statFile to be called 2 times but was called", statFileCalled) - } -} diff --git a/server/types/pac/parser/parse.go b/server/types/pac/parser/parse.go index e93a226d..cd4be247 100644 --- a/server/types/pac/parser/parse.go +++ b/server/types/pac/parser/parse.go @@ -1,7 +1,6 @@ package parser import ( - "fmt" "os" "path" "strings" @@ -38,6 +37,8 @@ func ParseAll() error { return errorx.Decorate(err, "failed to parse pacscripts") } + log.Info("pacscript parsing done. computing dependency graph") + for _, script := range loadedPacscripts { computeRequiredBy(script, loadedPacscripts) } @@ -46,12 +47,14 @@ func ParseAll() error { return s1.PackageName < s2.PackageName }) + log.Info("dependency graph done. setting up git updated-at dates") + if err := setLastUpdatedAt(loadedPacscripts); err != nil { return errorx.Decorate(err, "failed to set last updated at") } pacstore.Update(loadedPacscripts) - log.Info("successfully parsed %v (%v / %v) packages", types.Percent(float64(len(loadedPacscripts))/float64(len(pkgList))), len(loadedPacscripts), len(pkgList)) + log.Info("successfully loaded %v (%v / %v) packages", types.Percent(float64(len(loadedPacscripts))/float64(len(pkgList))), len(loadedPacscripts), len(pkgList)) return nil } @@ -81,45 +84,37 @@ func parsePacscriptFiles(names []string) ([]*pac.Script, error) { out, err := ParsePacscriptFile(config.GitClonePath, pacName) if config.Repology.Enabled { - if err := repology.Sync(&out); err != nil { + if err := repology.Sync(out); err != nil { log.Debug("failed to sync %v with repology. Error: %v", pacName, err) } } - return &out, err + return out, err }) return channels.ToSlice(outChan), nil } func readPacscriptFile(rootDir, name string) (scriptBytes []byte, fileName string, err error) { - fileName = fmt.Sprintf("%s.%s", name, consts.PACSCRIPT_FILE_EXTENSION) - scriptPath := path.Join(rootDir, "packages", name, fileName) + scriptPath := path.Join(rootDir, "packages", name, consts.SRCINFO_FILE_EXTENSION) scriptBytes, err = os.ReadFile(scriptPath) if err != nil { return nil, "", errorx.Decorate(err, "failed to read file '%v'", scriptPath) } - return scriptBytes, fileName, nil + return scriptBytes, consts.SRCINFO_FILE_EXTENSION, nil } -func ParsePacscriptFile(programsDirPath, name string) (pac.Script, error) { - pacshell, filename, err := readPacscriptFile(programsDirPath, name) - if err != nil { - return pac.Script{}, errorx.Decorate(err, "failed to read pacscript '%v'", name) - } - - pacshell = buildCustomFormatScript(pacshell) - - stdout, err := pacsh.ExecBash(config.TempDir, filename, pacshell) +func ParsePacscriptFile(programsDirPath, name string) (*pac.Script, error) { + srcInfoData, _, err := readPacscriptFile(programsDirPath, name) if err != nil { - return pac.Script{}, errorx.Decorate(err, "failed to execute pacscript '%v'", name) + return nil, errorx.Decorate(err, "failed to read pacscript '%v'", name) } - pacscript, err := pacsh.ParsePacOutput(stdout) + pacscript, err := pacsh.ParsePacOutput(srcInfoData) if err != nil { - return pac.Script{}, errorx.Decorate(err, "failed to parse pacscript '%v'", name) + return nil, errorx.Decorate(err, "failed to parse pacscript '%v'", name) } return pacscript, nil diff --git a/server/types/pac/parser/parse_test.go b/server/types/pac/parser/parse_test.go index 89fde1c4..3cdb0aab 100644 --- a/server/types/pac/parser/parse_test.go +++ b/server/types/pac/parser/parse_test.go @@ -7,6 +7,7 @@ import ( "path" "testing" + "pacstall.dev/webserver/types" "pacstall.dev/webserver/types/pac" "pacstall.dev/webserver/types/pac/parser" "pacstall.dev/webserver/types/pac/parser/pacsh" @@ -29,7 +30,24 @@ func assertEquals(t *testing.T, what string, expected interface{}, actual interf } } -func assertArrayEquals(t *testing.T, what string, expected []string, actual []string) { +func assertArrayEquals[T types.Equaller](t *testing.T, what string, expected []T, actual []T) { + if len(actual) == len(expected) && len(actual) == 0 { + return + } + + if len(actual) != len(expected) { + t.Errorf("pacscript.%v expected len '%v', got len '%v' (expected '%#v', got '%#v')", what, len(expected), len(actual), expected, actual) + return + } + + for idx := range expected { + if !expected[idx].Equals(actual[idx]) { + t.Errorf("pacscript.%v[%v] expected '%#v', got '%#v'", what, idx, expected, actual) + } + } +} + +func assertStringArrayEquals(t *testing.T, what string, expected []string, actual []string) { if len(actual) == len(expected) && len(actual) == 0 { return } @@ -46,9 +64,9 @@ func assertArrayEquals(t *testing.T, what string, expected []string, actual []st } } -func assertPacscriptEquals(t *testing.T, expected pac.Script, actual pac.Script) { +func assertPacscriptEquals(t *testing.T, expected *pac.Script, actual *pac.Script) { assertEquals(t, "package name", expected.PackageName, actual.PackageName) - assertArrayEquals(t, "maintainers", expected.Maintainers, actual.Maintainers) + assertStringArrayEquals(t, "maintainers", expected.Maintainers, actual.Maintainers) assertEquals(t, "description", expected.Description, actual.Description) assertEquals(t, "gives", expected.Gives, actual.Gives) if expected.Hash != nil && actual.Hash == nil { @@ -68,10 +86,8 @@ func assertPacscriptEquals(t *testing.T, expected pac.Script, actual pac.Script) assertArrayEquals(t, "build dependencies", expected.BuildDependencies, actual.BuildDependencies) assertArrayEquals(t, "optional dependencies", expected.OptionalDependencies, actual.OptionalDependencies) assertArrayEquals(t, "pacstall dependencies", expected.PacstallDependencies, actual.PacstallDependencies) - assertArrayEquals(t, "ppa", expected.PPA, actual.PPA) - assertArrayEquals(t, "patch", expected.Patch, actual.Patch) - assertArrayEquals(t, "required by", expected.RequiredBy, actual.RequiredBy) - assertArrayEquals(t, "repology", expected.Repology, actual.Repology) + assertStringArrayEquals(t, "required by", expected.RequiredBy, actual.RequiredBy) + assertStringArrayEquals(t, "repology", expected.Repology, actual.Repology) assertEquals(t, "update status", expected.UpdateStatus, actual.UpdateStatus) } @@ -121,7 +137,7 @@ func assertPacscriptMatchesSnapshot(t *testing.T, pkgname string) { return } - assertPacscriptEquals(t, *expected, actual) + assertPacscriptEquals(t, expected, actual) } func Test_PacscriptSnapshots(t *testing.T) { diff --git a/server/types/pac/parser/search.go b/server/types/pac/parser/search.go index 4e8c5776..f1b99407 100644 --- a/server/types/pac/parser/search.go +++ b/server/types/pac/parser/search.go @@ -37,7 +37,11 @@ func FilterPackages(packages []*pac.Script, filter, filterBy string) []*pac.Scri case "name": return filterByFunc(func(pi *pac.Script) bool { return strings.Contains(pi.PackageName, filter) || - strings.Contains(pi.Gives, filter) || + func(ps []pac.ArchDistroString, filter string) bool { + return array.Any(ps, func(it pac.ArchDistroString) bool { + return strings.Contains(it.Value, filter) + }) + }(pi.Gives, filter) || strings.Contains(pi.Description, filter) }) diff --git a/server/types/pac/script.go b/server/types/pac/script.go index b97f12da..00553732 100644 --- a/server/types/pac/script.go +++ b/server/types/pac/script.go @@ -1,6 +1,12 @@ package pac -import "time" +import ( + "time" + + "github.com/pacstall/go-srcinfo" + "pacstall.dev/webserver/types" + "pacstall.dev/webserver/types/array" +) type updateStatus struct { Unknown UpdateStatusValue @@ -20,27 +26,78 @@ var UpdateStatus = updateStatus{ type UpdateStatusValue = int +type ArchDistroString struct { + Arch string `json:"arch"` + Distro string `json:"distro"` + Value string `json:"value"` +} + +func (a ArchDistroString) Equals(b types.Equaller) bool { + other, ok := b.(ArchDistroString) + if !ok { + return false + } + + return a.Arch == other.Arch && a.Distro == other.Distro && a.Value == other.Value +} + type Script struct { - PrettyName string `json:"prettyName"` - Version string `json:"version"` - LatestVersion *string `json:"latestVersion"` - PackageName string `json:"packageName"` - Maintainers []string `json:"maintainers"` - Description string `json:"description"` - Source []string `json:"source"` - RuntimeDependencies []string `json:"runtimeDependencies"` - BuildDependencies []string `json:"buildDependencies"` - OptionalDependencies []string `json:"optionalDependencies"` - Conflicts []string `json:"conflicts"` - Breaks []string `json:"breaks"` - Gives string `json:"gives"` - Replaces []string `json:"replaces"` - Hash *string `json:"hash"` - PPA []string `json:"ppa"` - PacstallDependencies []string `json:"pacstallDependencies"` - Patch []string `json:"patch"` - Repology []string `json:"repology"` - RequiredBy []string `json:"requiredBy"` - LastUpdatedAt time.Time `json:"lastUpdatedAt"` - UpdateStatus int `json:"updateStatus"` // enum UpdateStatus + Architectures []string `json:"architectures"` + PrettyName string `json:"prettyName"` + Version string `json:"version"` + LatestVersion *string `json:"latestVersion"` + PackageName string `json:"packageName"` + Maintainers []string `json:"maintainers"` + Description string `json:"description"` + Source []ArchDistroString `json:"source"` + RuntimeDependencies []ArchDistroString `json:"runtimeDependencies"` + BuildDependencies []ArchDistroString `json:"buildDependencies"` + OptionalDependencies []ArchDistroString `json:"optionalDependencies"` + Conflicts []ArchDistroString `json:"conflicts"` + Breaks []ArchDistroString `json:"breaks"` + Gives []ArchDistroString `json:"gives"` + Replaces []ArchDistroString `json:"replaces"` + Hash *string `json:"hash"` + PacstallDependencies []ArchDistroString `json:"pacstallDependencies"` + Repology []string `json:"repology"` + RequiredBy []string `json:"requiredBy"` + LastUpdatedAt time.Time `json:"lastUpdatedAt"` + UpdateStatus int `json:"updateStatus"` // enum UpdateStatus +} + +func FromSrcInfo(info srcinfo.Srcinfo) *Script { + return &Script{ + Version: info.Version(), + LatestVersion: nil, + PackageName: info.Packages[0].Pkgname, + Maintainers: info.Maintainer, + Description: info.Pkgdesc, + Source: toArchDistroStrings(info.Source), + RuntimeDependencies: toArchDistroStrings(info.Depends), + BuildDependencies: toArchDistroStrings(info.MakeDepends), + OptionalDependencies: toArchDistroStrings(info.OptDepends), + Conflicts: toArchDistroStrings(info.Conflicts), + Breaks: toArchDistroStrings(info.Breaks), + Gives: toArchDistroStrings(info.Gives), + Replaces: toArchDistroStrings(info.Replaces), + PacstallDependencies: toArchDistroStrings(info.Pacdeps), + Architectures: info.Arch, + Repology: info.Repology, + RequiredBy: []string{}, + PrettyName: "", + } +} + +func toArchDistroStrings(ads []srcinfo.ArchDistroString) []ArchDistroString { + return array.SwitchMap(ads, func(it *array.Iterator[srcinfo.ArchDistroString]) ArchDistroString { + return toArchDistroString(it.Value) + }) +} + +func toArchDistroString(ads srcinfo.ArchDistroString) ArchDistroString { + return ArchDistroString{ + Arch: ads.Arch, + Distro: ads.Distro, + Value: ads.Value, + } } From de644622f338205b460aa7445ab95e1be074a10c Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Tue, 17 Sep 2024 20:08:41 +0300 Subject: [PATCH 02/17] feat: add missing fields --- server/model/connection.go | 27 +++++++++++-- server/types/pac/parser/parse_test.go | 7 ---- server/types/pac/script.go | 57 +++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/server/model/connection.go b/server/model/connection.go index c460120c..614d7f55 100644 --- a/server/model/connection.go +++ b/server/model/connection.go @@ -2,11 +2,13 @@ package model import ( "fmt" + "time" "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" "pacstall.dev/webserver/config" + "pacstall.dev/webserver/log" ) var database *gorm.DB = nil @@ -23,20 +25,39 @@ func Instance() *gorm.DB { return database } - db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), + err := retry(5, func() (err error) { + database, err = gorm.Open(mysql.Open(connectionString), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + + return }) if err != nil { panic(fmt.Sprintf("failed to connect database: %v", err)) } + log.Info("connected to database.") + defer postConnect() - database = db return database } +func retry(trials int, fn func() error) error { + var err error + for i := 0; i < trials; i += 1 { + if err = fn(); err != nil { + log.Warn("failed to connect to database. retrying...") + time.Sleep(3 * time.Second) + } else { + return nil + } + } + + return err +} + func postConnect() { database.AutoMigrate(&ShortenedLink{}) } diff --git a/server/types/pac/parser/parse_test.go b/server/types/pac/parser/parse_test.go index 3cdb0aab..984664ce 100644 --- a/server/types/pac/parser/parse_test.go +++ b/server/types/pac/parser/parse_test.go @@ -69,13 +69,6 @@ func assertPacscriptEquals(t *testing.T, expected *pac.Script, actual *pac.Scrip assertStringArrayEquals(t, "maintainers", expected.Maintainers, actual.Maintainers) assertEquals(t, "description", expected.Description, actual.Description) assertEquals(t, "gives", expected.Gives, actual.Gives) - if expected.Hash != nil && actual.Hash == nil { - t.Errorf("expected hash '%v', got nil", *expected.Hash) - } else if expected.Hash == nil && actual.Hash != nil { - t.Errorf("expected hash nil, got %v", *actual.Hash) - } else if expected.Hash != nil && actual.Hash != nil { - assertEquals(t, "hash", *expected.Hash, *actual.Hash) - } assertEquals(t, "version", expected.Version, actual.Version) assertArrayEquals(t, "breaks", expected.Breaks, actual.Breaks) assertArrayEquals(t, "conflicts", expected.Conflicts, actual.Conflicts) diff --git a/server/types/pac/script.go b/server/types/pac/script.go index 00553732..ea622472 100644 --- a/server/types/pac/script.go +++ b/server/types/pac/script.go @@ -27,8 +27,8 @@ var UpdateStatus = updateStatus{ type UpdateStatusValue = int type ArchDistroString struct { - Arch string `json:"arch"` - Distro string `json:"distro"` + Arch string `json:"arch,omitempty"` + Distro string `json:"distro,omitempty"` Value string `json:"value"` } @@ -53,16 +53,37 @@ type Script struct { RuntimeDependencies []ArchDistroString `json:"runtimeDependencies"` BuildDependencies []ArchDistroString `json:"buildDependencies"` OptionalDependencies []ArchDistroString `json:"optionalDependencies"` + CheckDependencies []ArchDistroString `json:"checkDependencies"` Conflicts []ArchDistroString `json:"conflicts"` Breaks []ArchDistroString `json:"breaks"` Gives []ArchDistroString `json:"gives"` Replaces []ArchDistroString `json:"replaces"` - Hash *string `json:"hash"` + Sha1Sums []ArchDistroString `json:"sha1sums"` + Sha224Sums []ArchDistroString `json:"sha224sums"` + Sha256Sums []ArchDistroString `json:"sha256sums"` + Sha384Sums []ArchDistroString `json:"sha384sums"` + Sha512Sums []ArchDistroString `json:"sha512sums"` + Md5Sums []ArchDistroString `json:"md5sums"` + Priority []ArchDistroString `json:"priority"` + Recommends []ArchDistroString `json:"recommends"` + Suggests []ArchDistroString `json:"suggests"` PacstallDependencies []ArchDistroString `json:"pacstallDependencies"` + Enhances []ArchDistroString `json:"enhances"` Repology []string `json:"repology"` RequiredBy []string `json:"requiredBy"` LastUpdatedAt time.Time `json:"lastUpdatedAt"` UpdateStatus int `json:"updateStatus"` // enum UpdateStatus + Changelog string `json:"changelog"` + Backup []string `json:"backup"` + Compatible []string `json:"compatible"` + Incompatible []string `json:"incompatible"` + Epoch string `json:"epoch"` + Install string `json:"install"` + License []string `json:"license"` + Mask []string `json:"mask"` + NoExtract []string `json:"noExtract"` + ValidPGPKeys []string `json:"validPgpKeys"` + Groups []string `json:"groups"` } func FromSrcInfo(info srcinfo.Srcinfo) *Script { @@ -84,7 +105,29 @@ func FromSrcInfo(info srcinfo.Srcinfo) *Script { Architectures: info.Arch, Repology: info.Repology, RequiredBy: []string{}, + Sha1Sums: toArchDistroStrings(info.SHA1Sums), + Sha224Sums: toArchDistroStrings(info.SHA224Sums), + Sha256Sums: toArchDistroStrings(info.SHA256Sums), + Sha384Sums: toArchDistroStrings(info.SHA384Sums), + Sha512Sums: toArchDistroStrings(info.SHA512Sums), PrettyName: "", + Changelog: info.Changelog, + Backup: orEmptyArray(info.Backup), + Compatible: orEmptyArray(info.Compatible), + Incompatible: orEmptyArray(info.Incompatible), + Epoch: info.Epoch, + Install: info.Install, + License: orEmptyArray(info.License), + Mask: orEmptyArray(info.Mask), + NoExtract: orEmptyArray(info.NoExtract), + ValidPGPKeys: orEmptyArray(info.ValidPGPKeys), + Groups: orEmptyArray(info.Groups), + Enhances: toArchDistroStrings(info.Enhances), + CheckDependencies: toArchDistroStrings(info.CheckDepends), + Md5Sums: toArchDistroStrings(info.MD5Sums), + Priority: toArchDistroStrings(info.Priority), + Suggests: toArchDistroStrings(info.Suggests), + Recommends: toArchDistroStrings(info.Recommends), } } @@ -101,3 +144,11 @@ func toArchDistroString(ads srcinfo.ArchDistroString) ArchDistroString { Value: ads.Value, } } + +func orEmptyArray[T interface{}](items []T) []T { + if items == nil { + return []T{} + } + + return items +} From 375a8af59e4aeee5680c45f8ccb66fd900b3d161 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Tue, 17 Sep 2024 20:27:33 +0300 Subject: [PATCH 03/17] feat: update client models --- .../package-details/MinimalPackageTable.tsx | 8 ++-- .../MinimalPackageTableRow.tsx | 4 +- .../package-details/PackageDetailsPage.tsx | 4 +- .../components/packages/PackageTableRow.tsx | 4 +- client/src/types/package-dependencies.ts | 10 ++-- client/src/types/package-info.ts | 48 ++++++++++++++----- 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/client/src/components/package-details/MinimalPackageTable.tsx b/client/src/components/package-details/MinimalPackageTable.tsx index bc91289e..54ac5198 100644 --- a/client/src/components/package-details/MinimalPackageTable.tsx +++ b/client/src/components/package-details/MinimalPackageTable.tsx @@ -1,10 +1,10 @@ import { Table, Tbody, Th, Thead, Tr } from '@chakra-ui/react' import { FC } from 'react' -import PackageInfo from '../../types/package-info' +import { ArchDistroString } from '../../types/package-info' import MinimalPackageTableRow from './MinimalPackageTableRow' import { useTranslation } from 'react-i18next' -const MinimalPackageTable: FC<{ packages: (PackageInfo | string)[] }> = ({ +const MinimalPackageTable: FC<{ packages: (ArchDistroString | string)[] }> = ({ packages, }) => { const { t } = useTranslation() @@ -23,10 +23,10 @@ const MinimalPackageTable: FC<{ packages: (PackageInfo | string)[] }> = ({ ))} diff --git a/client/src/components/package-details/MinimalPackageTableRow.tsx b/client/src/components/package-details/MinimalPackageTableRow.tsx index fdc70247..7a053376 100644 --- a/client/src/components/package-details/MinimalPackageTableRow.tsx +++ b/client/src/components/package-details/MinimalPackageTableRow.tsx @@ -11,13 +11,13 @@ import { useTranslation } from 'react-i18next' import { Link as Rlink } from 'react-router-dom' const getDescription = (nameWithDescription: string): string | null => { - return nameWithDescription.includes(':') + return nameWithDescription?.includes(':') ? nameWithDescription.split(':')[1].trim() : null } const getName = (nameWithDescription: string): string => { - return nameWithDescription.includes(':') + return nameWithDescription?.includes(':') ? nameWithDescription.split(':')[0] : nameWithDescription } diff --git a/client/src/components/package-details/PackageDetailsPage.tsx b/client/src/components/package-details/PackageDetailsPage.tsx index 9349b815..d00aa32b 100644 --- a/client/src/components/package-details/PackageDetailsPage.tsx +++ b/client/src/components/package-details/PackageDetailsPage.tsx @@ -1,7 +1,7 @@ import { Container, UseDisclosureProps } from '@chakra-ui/react' import { FC } from 'react' import { Helmet } from 'react-helmet' -import PackageInfo from '../../types/package-info' +import PackageInfo, { ArchDistroString } from '../../types/package-info' import HowToInstall from './HowToInstall' import PackageDependenciesModal from './PackageDependenciesModal' import PackageDetailsHeader from './PackageDetailsHeader' @@ -10,7 +10,7 @@ import PackageRequiredByModal from './PackageRequiredByModal' type PackageDetailsPageProps = { data: PackageInfo - allDependencies: string[] + allDependencies: ArchDistroString[] isMobile: boolean requiredByModal: UseDisclosureProps dependenciesModal: UseDisclosureProps diff --git a/client/src/components/packages/PackageTableRow.tsx b/client/src/components/packages/PackageTableRow.tsx index 3f71d997..ecb8843c 100644 --- a/client/src/components/packages/PackageTableRow.tsx +++ b/client/src/components/packages/PackageTableRow.tsx @@ -53,7 +53,7 @@ const PackageTableRow: FC<{ pkg: PackageInfo; disabled?: boolean }> = ({ = ({ } > - {pkg.maintainers + {(pkg.maintainers ?? []) .map(maintainer => maintainer.split('<')[0].trim()) .join(', ') || t('packageDetails.orphaned')} diff --git a/client/src/types/package-dependencies.ts b/client/src/types/package-dependencies.ts index dc0836cc..af9cb7e2 100644 --- a/client/src/types/package-dependencies.ts +++ b/client/src/types/package-dependencies.ts @@ -1,8 +1,8 @@ -import PackageInfo from './package-info' +import PackageInfo, { ArchDistroString } from './package-info' export default interface PackageDependencies { - runtimeDependencies: (PackageInfo | string)[] - buildDependencies: (PackageInfo | string)[] - optionalDependencies: (PackageInfo | string)[] - pacstallDependencies: (PackageInfo | string)[] + runtimeDependencies: (ArchDistroString | string)[] + buildDependencies: (ArchDistroString | string)[] + optionalDependencies: (ArchDistroString | string)[] + pacstallDependencies: (ArchDistroString | string)[] } diff --git a/client/src/types/package-info.ts b/client/src/types/package-info.ts index 9309379e..126d0d9c 100644 --- a/client/src/types/package-info.ts +++ b/client/src/types/package-info.ts @@ -1,24 +1,50 @@ export default interface PackageInfo { + architectures: string[] version: string packageName: string maintainers: string[] description: string - source: string[] - runtimeDependencies: string[] - buildDependencies: string[] - optionalDependencies: string[] - conflicts: string[] - gives: string - replaces: string[] - hash?: string - ppa: string[] - pacstallDependencies: string[] - patch: string[] + source: ArchDistroString[] + runtimeDependencies: ArchDistroString[] + buildDependencies: ArchDistroString[] + optionalDependencies: ArchDistroString[] + checkDependencies: ArchDistroString[] + pacstallDependencies: ArchDistroString[] + conflicts: ArchDistroString[] + gives: ArchDistroString + replaces: ArchDistroString[] + sha1sums: ArchDistroString[] + sha224sums: ArchDistroString[] + sha256sums: ArchDistroString[] + sha384sums: ArchDistroString[] + sha512sums: ArchDistroString[] + md5sums: ArchDistroString[] + priority: ArchDistroString[] requiredBy: string[] + suggests: ArchDistroString[] + recommends: ArchDistroString[] latestVersion?: string prettyName: string updateStatus: UpdateStatus lastUpdatedAt: string + enhances: ArchDistroString[] + changelog: string + backup: string[] + compatible: string[] + incompatible: string[] + epoch: string + install: string + license: string[] + mask: string[] + noExtract: string[] + validPgpKeys: string[] + groups: string[] +} + +export interface ArchDistroString { + arch?: string + distro?: string + value: string } export enum UpdateStatus { From 580f24256508d1ba6944881e53c74deefe7efe40 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Tue, 17 Sep 2024 20:28:15 +0300 Subject: [PATCH 04/17] lint: code styles --- .../src/components/package-details/MinimalPackageTable.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/src/components/package-details/MinimalPackageTable.tsx b/client/src/components/package-details/MinimalPackageTable.tsx index 54ac5198..a15a0024 100644 --- a/client/src/components/package-details/MinimalPackageTable.tsx +++ b/client/src/components/package-details/MinimalPackageTable.tsx @@ -22,10 +22,7 @@ const MinimalPackageTable: FC<{ packages: (ArchDistroString | string)[] }> = ({ {packages.map((pkg, i) => ( ))} From 3cf80f70b01f7f001675e6a15bb5631e22abbd39 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Tue, 17 Sep 2024 21:24:59 +0300 Subject: [PATCH 05/17] fix: repology api recipe url --- server/server/api/repology/types.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/server/api/repology/types.go b/server/server/api/repology/types.go index eb1600e4..e1c12b79 100644 --- a/server/server/api/repology/types.go +++ b/server/server/api/repology/types.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "pacstall.dev/webserver/consts" "pacstall.dev/webserver/types/pac" ) @@ -39,7 +38,7 @@ func newRepologyPackage(p *pac.Script) repologyPackage { Version: p.Version, URL: source, Type: getType(p), - RecipeURL: fmt.Sprintf("https://raw.githubusercontent.com/pacstall/pacstall-programs/master/packages/%s/%s.%s", p.PackageName, p.PackageName, consts.SRCINFO_FILE_EXTENSION), + RecipeURL: fmt.Sprintf("https://raw.githubusercontent.com/pacstall/pacstall-programs/master/packages/%s/%s.pacscript", p.PackageName, p.PackageName), PackageDetailsURL: fmt.Sprintf("https://pacstall.dev/packages/%s", p.PackageName), } } From c3890ecc600905658ca29f38cbf558dbb3b456ca Mon Sep 17 00:00:00 2001 From: Elsie Date: Wed, 18 Sep 2024 10:21:47 -0400 Subject: [PATCH 06/17] upd(Dockerfile): remove no-longer-needed dependency --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 84317881..ff1aac44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ COPY --from=client /root/client/dist/ /root/client/dist/ COPY --from=server /root/server/dist/ /root/server/dist/ COPY ./Makefile ./Makefile -RUN apt update && apt install make git jo jq -y +RUN apt update && apt install make git jq -y RUN make dist \ && rm -rf server client From cb0070067f0b58d749263cc78521001b4a6d26a7 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Wed, 18 Sep 2024 21:39:44 +0300 Subject: [PATCH 07/17] feat: add git version --- server/server/api/repology/types.go | 19 +------------- server/types/pac/parser/pacsh/git_version.go | 22 +++++++++++++++++ server/types/pac/parser/pacsh/pretty-name.go | 12 +++------ server/types/pac/parser/parse.go | 12 +++++++++ server/types/pac/script.go | 11 +++++++++ server/types/pkgtype.go | 26 ++++++++++++++++++++ 6 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 server/types/pac/parser/pacsh/git_version.go create mode 100644 server/types/pkgtype.go diff --git a/server/server/api/repology/types.go b/server/server/api/repology/types.go index e1c12b79..de81339c 100644 --- a/server/server/api/repology/types.go +++ b/server/server/api/repology/types.go @@ -37,19 +37,12 @@ func newRepologyPackage(p *pac.Script) repologyPackage { Maintainer: getMaintainer(p), Version: p.Version, URL: source, - Type: getType(p), + Type: string(p.Type()), RecipeURL: fmt.Sprintf("https://raw.githubusercontent.com/pacstall/pacstall-programs/master/packages/%s/%s.pacscript", p.PackageName, p.PackageName), PackageDetailsURL: fmt.Sprintf("https://pacstall.dev/packages/%s", p.PackageName), } } -var pacTypes = map[string]string{ - "-deb": "Debian Native", - "-git": "Source Code", - "-bin": "Precompiled", - "-app": "AppImage", -} - func getMaintainer(p *pac.Script) maintainerDetails { maintainer := "" if len(p.Maintainers) > 0 { @@ -71,13 +64,3 @@ func getMaintainer(p *pac.Script) maintainerDetails { Email: &email, } } - -func getType(p *pac.Script) string { - for suffix, kind := range pacTypes { - if strings.HasSuffix(p.PackageName, suffix) { - return kind - } - } - - return pacTypes["-git"] -} diff --git a/server/types/pac/parser/pacsh/git_version.go b/server/types/pac/parser/pacsh/git_version.go new file mode 100644 index 00000000..56e1b97f --- /dev/null +++ b/server/types/pac/parser/pacsh/git_version.go @@ -0,0 +1,22 @@ +package pacsh + +import ( + "github.com/joomcode/errorx" + "pacstall.dev/webserver/types/array" + "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/types/pac/parser/pacsh/internal" +) + +func ApplyGitVersion(p *pac.Script) error { + sources := internal.NewGitSources(array.SwitchMap(p.Source, func(it *array.Iterator[pac.ArchDistroString]) string { + return it.Value.Value + })) + + version, err := sources.ParseGitPackageVersion() + if err != nil { + return errorx.Decorate(err, "failed to parse git version for package '%s'", p.PackageName) + } + + p.Version = version + return nil +} diff --git a/server/types/pac/parser/pacsh/pretty-name.go b/server/types/pac/parser/pacsh/pretty-name.go index dc071fdd..d1e12030 100644 --- a/server/types/pac/parser/pacsh/pretty-name.go +++ b/server/types/pac/parser/pacsh/pretty-name.go @@ -3,16 +3,10 @@ package pacsh import ( "strings" + "pacstall.dev/webserver/types" "pacstall.dev/webserver/types/pac" ) -var pacTypes = map[string]string{ - "-deb": "Debian Native", - "-git": "Source Code", - "-bin": "Precompiled", - "-app": "AppImage", -} - func getPrettyName(p *pac.Script) string { name := "" @@ -20,8 +14,8 @@ func getPrettyName(p *pac.Script) string { name = p.PackageName } - for suffix := range pacTypes { - if strings.HasSuffix(name, suffix) { + for suffix := range types.PackageTypeSuffixToPackageTypeName { + if strings.HasSuffix(name, string(suffix)) { name = name[0 : len(name)-len(suffix)] } } diff --git a/server/types/pac/parser/parse.go b/server/types/pac/parser/parse.go index cd4be247..6d4f6101 100644 --- a/server/types/pac/parser/parse.go +++ b/server/types/pac/parser/parse.go @@ -21,6 +21,7 @@ import ( ) const PACKAGE_LIST_FILE_NAME = "./packagelist" +const MAX_GIT_VERSION_CONCURRENCY = 5 func ParseAll() error { if err := git.RefreshPrograms(config.GitClonePath, config.GitURL, config.PacstallPrograms.Branch); err != nil { @@ -53,6 +54,17 @@ func ParseAll() error { return errorx.Decorate(err, "failed to set last updated at") } + log.Info("updated-at dates done. fetching git versions") + + gitPacscripts := array.Filter(loadedPacscripts, func(it *array.Iterator[*pac.Script]) bool { + return strings.HasSuffix(it.Value.PackageName, string(types.PACKAGE_TYPE_SUFFIX_GIT)) + }) + + batch.Run(MAX_GIT_VERSION_CONCURRENCY, gitPacscripts, func(p *pac.Script) (interface{}, error) { + err := pacsh.ApplyGitVersion(p) + return nil, err + }) + pacstore.Update(loadedPacscripts) log.Info("successfully loaded %v (%v / %v) packages", types.Percent(float64(len(loadedPacscripts))/float64(len(pkgList))), len(loadedPacscripts), len(pkgList)) diff --git a/server/types/pac/script.go b/server/types/pac/script.go index ea622472..8e73dbb8 100644 --- a/server/types/pac/script.go +++ b/server/types/pac/script.go @@ -1,6 +1,7 @@ package pac import ( + "strings" "time" "github.com/pacstall/go-srcinfo" @@ -86,6 +87,16 @@ type Script struct { Groups []string `json:"groups"` } +func (p *Script) Type() types.PackageTypeName { + for suffix, name := range types.PackageTypeSuffixToPackageTypeName { + if strings.HasSuffix(p.PackageName, string(suffix)) { + return name + } + } + + return types.PackageTypeSuffixToPackageTypeName["-git"] +} + func FromSrcInfo(info srcinfo.Srcinfo) *Script { return &Script{ Version: info.Version(), diff --git a/server/types/pkgtype.go b/server/types/pkgtype.go new file mode 100644 index 00000000..765ce8dc --- /dev/null +++ b/server/types/pkgtype.go @@ -0,0 +1,26 @@ +package types + +type PackageTypeName string + +const ( + PACKAGE_TYPE_DEB PackageTypeName = "Debian Native" + PACKAGE_TYPE_GIT PackageTypeName = "Source Code" + PACKAGE_TYPE_BIN PackageTypeName = "Precompiled" + PACKAGE_TYPE_APP PackageTypeName = "AppImage" +) + +type PackageTypeSuffix string + +const ( + PACKAGE_TYPE_SUFFIX_DEB PackageTypeSuffix = "-deb" + PACKAGE_TYPE_SUFFIX_GIT PackageTypeSuffix = "-git" + PACKAGE_TYPE_SUFFIX_BIN PackageTypeSuffix = "-bin" + PACKAGE_TYPE_SUFFIX_APP PackageTypeSuffix = "-app" +) + +var PackageTypeSuffixToPackageTypeName = map[PackageTypeSuffix]PackageTypeName{ + PACKAGE_TYPE_SUFFIX_DEB: PACKAGE_TYPE_DEB, + PACKAGE_TYPE_SUFFIX_GIT: PACKAGE_TYPE_GIT, + PACKAGE_TYPE_SUFFIX_BIN: PACKAGE_TYPE_BIN, + PACKAGE_TYPE_SUFFIX_APP: PACKAGE_TYPE_APP, +} From 5abd301561a7cf69a6edd7d4e52337def8433d05 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Wed, 18 Sep 2024 22:37:56 +0300 Subject: [PATCH 08/17] feat: fix repology exporter and default package update status --- server/repology/internal/api.go | 12 +++++++++--- server/repology/internal/exporter.go | 24 +++++++++++++++++++++--- server/repology/scheduler.go | 2 +- server/types/pac/parser/parse.go | 2 +- server/types/pac/script.go | 1 + 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/server/repology/internal/api.go b/server/repology/internal/api.go index abaa95f2..6201a441 100644 --- a/server/repology/internal/api.go +++ b/server/repology/internal/api.go @@ -5,6 +5,8 @@ import ( "io" "net/http" "net/url" + + "github.com/joomcode/errorx" ) const _USER_AGENT = "Pacstall/WebServer/Exporter" @@ -23,17 +25,21 @@ func getProjectSearch(projectName string) (RepologyApiProjectSearchResponse, err resp, err := http.DefaultClient.Do(&request) if err != nil { - return response, err + return response, errorx.Decorate(err, "http request failed %+v", request) } body, err := io.ReadAll(resp.Body) if err != nil { - return response, err + return response, errorx.Decorate(err, "failed to read request body") + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return response, errorx.RejectedOperation.New("http request failed with status code %v. body \n%v\n", resp.StatusCode, string(body)) } err = json.Unmarshal(body, &response) if err != nil { - return response, err + return response, errorx.Decorate(err, "failed to unmarshal response body '%v'", string(body)) } return response, err diff --git a/server/repology/internal/exporter.go b/server/repology/internal/exporter.go index 7a72cee7..a55ecd02 100644 --- a/server/repology/internal/exporter.go +++ b/server/repology/internal/exporter.go @@ -6,11 +6,14 @@ import ( "strings" "time" + "github.com/joomcode/errorx" "gorm.io/gorm" "pacstall.dev/webserver/log" "pacstall.dev/webserver/model" ) +const RETRY_COUNT = 5 + func ExportRepologyDatabase(db *gorm.DB) error { err := migrateTables(db) if err != nil { @@ -20,7 +23,7 @@ func ExportRepologyDatabase(db *gorm.DB) error { it := 1 lastProjectName := "" - const REPOLOGY_PROJECT_FETCH_THROTTLE = 400 * time.Millisecond + const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second lastRepoFetch := time.Now() @@ -30,9 +33,24 @@ func ExportRepologyDatabase(db *gorm.DB) error { } log.Debug("page %v | cursor at: %v", it, lastProjectName) - projectPage, err := getProjectSearch(lastProjectName) + + var projectPage map[string][]RepologyApiProject + var err error + + retry: + for i := 0; i < RETRY_COUNT; i += 1 { + projectPage, err = getProjectSearch(lastProjectName) + if err == nil { + break retry + } + + retryDelay := time.Duration(i+1) * REPOLOGY_PROJECT_FETCH_THROTTLE + log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) + time.Sleep(retryDelay) + } + if err != nil { - return errors.Join(errors.New("failed to fetch repology project page"), err) + return errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) } lastRepoFetch = time.Now() diff --git a/server/repology/scheduler.go b/server/repology/scheduler.go index 74324a07..b917befb 100644 --- a/server/repology/scheduler.go +++ b/server/repology/scheduler.go @@ -15,7 +15,7 @@ func ScheduleRefresh(every time.Duration) { log.Info("refreshing Repology database...") err := ExportRepologyDatabase(db) if err != nil { - log.Error("failed to export Repology projects: %v", err) + log.Error("failed to export Repology projects: %+v", err) } else { log.Info("repology database refreshed successfully") } diff --git a/server/types/pac/parser/parse.go b/server/types/pac/parser/parse.go index 6d4f6101..689d8cea 100644 --- a/server/types/pac/parser/parse.go +++ b/server/types/pac/parser/parse.go @@ -97,7 +97,7 @@ func parsePacscriptFiles(names []string) ([]*pac.Script, error) { if config.Repology.Enabled { if err := repology.Sync(out); err != nil { - log.Debug("failed to sync %v with repology. Error: %v", pacName, err) + log.Debug("failed to sync %v with repology. Error: %+v", pacName, err) } } diff --git a/server/types/pac/script.go b/server/types/pac/script.go index 8e73dbb8..b4d6674d 100644 --- a/server/types/pac/script.go +++ b/server/types/pac/script.go @@ -139,6 +139,7 @@ func FromSrcInfo(info srcinfo.Srcinfo) *Script { Priority: toArchDistroStrings(info.Priority), Suggests: toArchDistroStrings(info.Suggests), Recommends: toArchDistroStrings(info.Recommends), + UpdateStatus: UpdateStatus.Unknown, } } From 4bc3088ccd7bb01fa4113cab46b4d0cdd6a58f2b Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Fri, 20 Sep 2024 16:10:54 +0300 Subject: [PATCH 09/17] fix: allow longer versions on package search page --- client/src/components/packages/PackageTableRow.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/packages/PackageTableRow.tsx b/client/src/components/packages/PackageTableRow.tsx index ecb8843c..da198715 100644 --- a/client/src/components/packages/PackageTableRow.tsx +++ b/client/src/components/packages/PackageTableRow.tsx @@ -81,7 +81,7 @@ const PackageTableRow: FC<{ pkg: PackageInfo; disabled?: boolean }> = ({ From 080be6b98664c41ee6c4c8e505e51e8c467965a2 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Fri, 20 Sep 2024 16:38:45 +0300 Subject: [PATCH 10/17] fix: add pkgrel to git version --- server/types/pac/parser/pacsh/git_version.go | 7 ++++++- server/types/pac/script.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/types/pac/parser/pacsh/git_version.go b/server/types/pac/parser/pacsh/git_version.go index 56e1b97f..c40d00f9 100644 --- a/server/types/pac/parser/pacsh/git_version.go +++ b/server/types/pac/parser/pacsh/git_version.go @@ -17,6 +17,11 @@ func ApplyGitVersion(p *pac.Script) error { return errorx.Decorate(err, "failed to parse git version for package '%s'", p.PackageName) } - p.Version = version + if p.Epoch != "" { + p.Version = p.Epoch + ":" + version + "-" + p.Release + } else { + p.Version = version + "-" + p.Release + } + return nil } diff --git a/server/types/pac/script.go b/server/types/pac/script.go index b4d6674d..96b43298 100644 --- a/server/types/pac/script.go +++ b/server/types/pac/script.go @@ -46,6 +46,7 @@ type Script struct { Architectures []string `json:"architectures"` PrettyName string `json:"prettyName"` Version string `json:"version"` + Release string `json:"release"` LatestVersion *string `json:"latestVersion"` PackageName string `json:"packageName"` Maintainers []string `json:"maintainers"` @@ -100,6 +101,7 @@ func (p *Script) Type() types.PackageTypeName { func FromSrcInfo(info srcinfo.Srcinfo) *Script { return &Script{ Version: info.Version(), + Release: info.Pkgrel, LatestVersion: nil, PackageName: info.Packages[0].Pkgname, Maintainers: info.Maintainer, From b516196bfd4607c050d0d366b58212e60a1f9ec6 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Sat, 21 Sep 2024 21:03:06 +0300 Subject: [PATCH 11/17] fix: git version channel never exhausted --- server/types/pac/parser/parallelism/channels/exhaust.go | 6 ++++++ server/types/pac/parser/parse.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 server/types/pac/parser/parallelism/channels/exhaust.go diff --git a/server/types/pac/parser/parallelism/channels/exhaust.go b/server/types/pac/parser/parallelism/channels/exhaust.go new file mode 100644 index 00000000..f9d63c09 --- /dev/null +++ b/server/types/pac/parser/parallelism/channels/exhaust.go @@ -0,0 +1,6 @@ +package channels + +func Exhaust[T any](in <-chan T) { + for range in { + } +} diff --git a/server/types/pac/parser/parse.go b/server/types/pac/parser/parse.go index 689d8cea..26eca283 100644 --- a/server/types/pac/parser/parse.go +++ b/server/types/pac/parser/parse.go @@ -60,10 +60,10 @@ func ParseAll() error { return strings.HasSuffix(it.Value.PackageName, string(types.PACKAGE_TYPE_SUFFIX_GIT)) }) - batch.Run(MAX_GIT_VERSION_CONCURRENCY, gitPacscripts, func(p *pac.Script) (interface{}, error) { + channels.Exhaust(batch.Run(MAX_GIT_VERSION_CONCURRENCY, gitPacscripts, func(p *pac.Script) (interface{}, error) { err := pacsh.ApplyGitVersion(p) return nil, err - }) + })) pacstore.Update(loadedPacscripts) log.Info("successfully loaded %v (%v / %v) packages", types.Percent(float64(len(loadedPacscripts))/float64(len(pkgList))), len(loadedPacscripts), len(pkgList)) From a7080733055728310f13864a6360c76c174089a0 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Mon, 23 Sep 2024 15:47:46 +0300 Subject: [PATCH 12/17] feat: reorganize packages --- server/.gitignore | 1 + server/Makefile | 80 +++----- server/bin/webserver/main.go | 84 -------- server/cmd/pacnexus/main.go | 86 ++++++++ server/cmd/pacsight/main.go | 55 ++++++ server/config/build/vars.go | 14 -- server/config/env.go | 49 ----- server/config/vars.go | 21 -- server/internal/pacnexus/config/env.go | 108 ++++++++++ .../{ => internal/pacnexus}/consts/consts.go | 0 .../pacnexus}/model/connection.go | 25 +-- .../pacnexus}/model/repology_project.go | 0 .../model/repology_project_provider.go | 0 .../pacnexus}/model/shortened_link.go | 0 .../server/api/pacscripts/dependencies.go | 8 +- .../server/api/pacscripts/package.go | 8 +- .../server/api/pacscripts/package_list.go | 10 +- .../server/api/pacscripts/required_by.go | 8 +- .../pacnexus}/server/api/repology/repology.go | 8 +- .../pacnexus}/server/api/repology/types.go | 2 +- .../server/api/url_shortener/url_shortener.go | 11 +- .../{ => internal/pacnexus}/server/headers.go | 2 +- .../pacnexus}/server/health_check.go | 2 +- .../pacnexus}/server/query/main.go | 0 .../{ => internal/pacnexus}/server/sitemap.go | 2 +- server/{ => internal/pacnexus}/server/spa.go | 2 +- .../pacnexus}/server/ssr/pacscript/main.go | 0 .../pacnexus}/server/ssr/pacscript/package.go | 10 +- .../server/ssr/pacscript/packages.go | 2 +- .../pacnexus}/server/ssr/templates.go | 0 .../pacnexus}/server/webserver.go | 13 +- .../pacnexus}/types/pac/pacstore/store.go | 4 +- .../pacnexus}/types/pac/parser/git/lib.go | 2 +- .../types/pac/parser/last_updated.go | 12 +- .../pacnexus}/types/pac/parser/pacscript.go | 4 +- .../types/pac/parser/pacscript_list.go | 2 +- .../types/pac/parser/pacsh/exec_sh.go | 2 +- .../types/pac/parser/pacsh/git_version.go | 6 +- .../pac/parser/pacsh/internal/git_version.go | 4 +- .../parser/pacsh/internal/git_version_test.go | 2 +- .../pac/parser/pacsh/parse_pac_output.go | 2 +- .../types/pac/parser/pacsh/pretty-name.go | 4 +- .../types/pac/parser/pacsh/temp_dir.go | 0 .../types/pac/parser/pacsh/temp_exec.go | 2 +- .../types/pac/parser/parallelism/batch/run.go | 2 +- .../pac/parser/parallelism/batch/run_test.go | 6 +- .../parser/parallelism/channels/exhaust.go | 0 .../parser/parallelism/channels/to_slice.go | 0 .../parallelism/channels/to_slice_test.go | 0 .../pac/parser/parallelism/timeout/timeout.go | 0 .../pacnexus}/types/pac/parser/parse.go | 54 ++--- .../pacnexus/types/pac/parser/scheduler.go | 28 +++ .../pacnexus}/types/pac/parser/search.go | 4 +- .../pacnexus/types/pac/parser}/sync.go | 10 +- .../pacnexus}/types/pac/script.go | 4 +- server/internal/pacsight/config/env.go | 28 +++ server/internal/pacsight/repology/exporter.go | 5 + server/internal/pacsight/repology/fetch.go | 97 +++++++++ .../internal/pacsight/repology/scheduler.go | 67 +++++++ .../pacsight/repology/types}/api.go | 7 +- .../pacsight/repology/types/exporter.go | 98 +++++++++ server/internal/pacsight/rpcall/service.go | 26 +++ server/{types => pkg/common}/array/array.go | 0 .../{types => pkg/common}/array/array_test.go | 0 server/{types => pkg/common}/array/sort.go | 0 server/pkg/common/config/build/vars.go | 4 + server/pkg/common/config/vars.go | 8 + server/pkg/common/env/env.go | 119 +++++++++++ server/{config => pkg/common/env}/util.go | 40 +++- server/{ => pkg/common}/log/discord.go | 0 server/{ => pkg/common}/log/lib.go | 48 ----- server/pkg/common/pacsight/rpc.go | 42 ++++ server/{ => pkg/common}/types/equals.go | 0 server/{ => pkg/common}/types/percentage.go | 0 server/{ => pkg/common}/types/pkgtype.go | 0 .../common/types/repology.go} | 2 +- server/repology/exporter.go | 5 - server/repology/fetch.go | 78 -------- server/repology/internal/exporter.go | 186 ------------------ server/repology/lib.go | 18 -- server/repology/scheduler.go | 25 --- server/types/pac/parser/parse_test.go | 151 -------------- server/types/pac/parser/scheduler.go | 27 --- 83 files changed, 963 insertions(+), 883 deletions(-) delete mode 100644 server/bin/webserver/main.go create mode 100644 server/cmd/pacnexus/main.go create mode 100644 server/cmd/pacsight/main.go delete mode 100644 server/config/build/vars.go delete mode 100644 server/config/env.go delete mode 100644 server/config/vars.go create mode 100644 server/internal/pacnexus/config/env.go rename server/{ => internal/pacnexus}/consts/consts.go (100%) rename server/{ => internal/pacnexus}/model/connection.go (60%) rename server/{ => internal/pacnexus}/model/repology_project.go (100%) rename server/{ => internal/pacnexus}/model/repology_project_provider.go (100%) rename server/{ => internal/pacnexus}/model/shortened_link.go (100%) rename server/{ => internal/pacnexus}/server/api/pacscripts/dependencies.go (85%) rename server/{ => internal/pacnexus}/server/api/pacscripts/package.go (73%) rename server/{ => internal/pacnexus}/server/api/pacscripts/package_list.go (89%) rename server/{ => internal/pacnexus}/server/api/pacscripts/required_by.go (79%) rename server/{ => internal/pacnexus}/server/api/repology/repology.go (74%) rename server/{ => internal/pacnexus}/server/api/repology/types.go (97%) rename server/{ => internal/pacnexus}/server/api/url_shortener/url_shortener.go (83%) rename server/{ => internal/pacnexus}/server/headers.go (95%) rename server/{ => internal/pacnexus}/server/health_check.go (95%) rename server/{ => internal/pacnexus}/server/query/main.go (100%) rename server/{ => internal/pacnexus}/server/sitemap.go (96%) rename server/{ => internal/pacnexus}/server/spa.go (96%) rename server/{ => internal/pacnexus}/server/ssr/pacscript/main.go (100%) rename server/{ => internal/pacnexus}/server/ssr/pacscript/package.go (84%) rename server/{ => internal/pacnexus}/server/ssr/pacscript/packages.go (88%) rename server/{ => internal/pacnexus}/server/ssr/templates.go (100%) rename server/{ => internal/pacnexus}/server/webserver.go (80%) rename server/{ => internal/pacnexus}/types/pac/pacstore/store.go (88%) rename server/{ => internal/pacnexus}/types/pac/parser/git/lib.go (98%) rename server/{ => internal/pacnexus}/types/pac/parser/last_updated.go (87%) rename server/{ => internal/pacnexus}/types/pac/parser/pacscript.go (84%) rename server/{ => internal/pacnexus}/types/pac/parser/pacscript_list.go (76%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/exec_sh.go (94%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/git_version.go (76%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/internal/git_version.go (96%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/internal/git_version_test.go (97%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/parse_pac_output.go (83%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/pretty-name.go (86%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/temp_dir.go (100%) rename server/{ => internal/pacnexus}/types/pac/parser/pacsh/temp_exec.go (96%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/batch/run.go (93%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/batch/run_test.go (72%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/channels/exhaust.go (100%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/channels/to_slice.go (100%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/channels/to_slice_test.go (100%) rename server/{ => internal/pacnexus}/types/pac/parser/parallelism/timeout/timeout.go (100%) rename server/{ => internal/pacnexus}/types/pac/parser/parse.go (60%) create mode 100644 server/internal/pacnexus/types/pac/parser/scheduler.go rename server/{ => internal/pacnexus}/types/pac/parser/search.go (96%) rename server/{repology => internal/pacnexus/types/pac/parser}/sync.go (89%) rename server/{ => internal/pacnexus}/types/pac/script.go (98%) create mode 100644 server/internal/pacsight/config/env.go create mode 100644 server/internal/pacsight/repology/exporter.go create mode 100644 server/internal/pacsight/repology/fetch.go create mode 100644 server/internal/pacsight/repology/scheduler.go rename server/{repology/internal => internal/pacsight/repology/types}/api.go (83%) create mode 100644 server/internal/pacsight/repology/types/exporter.go create mode 100644 server/internal/pacsight/rpcall/service.go rename server/{types => pkg/common}/array/array.go (100%) rename server/{types => pkg/common}/array/array_test.go (100%) rename server/{types => pkg/common}/array/sort.go (100%) create mode 100644 server/pkg/common/config/build/vars.go create mode 100644 server/pkg/common/config/vars.go create mode 100644 server/pkg/common/env/env.go rename server/{config => pkg/common/env}/util.go (60%) rename server/{ => pkg/common}/log/discord.go (100%) rename server/{ => pkg/common}/log/lib.go (53%) create mode 100644 server/pkg/common/pacsight/rpc.go rename server/{ => pkg/common}/types/equals.go (100%) rename server/{ => pkg/common}/types/percentage.go (100%) rename server/{ => pkg/common}/types/pkgtype.go (100%) rename server/{repology/internal/api_types.go => pkg/common/types/repology.go} (97%) delete mode 100644 server/repology/exporter.go delete mode 100644 server/repology/fetch.go delete mode 100644 server/repology/internal/exporter.go delete mode 100644 server/repology/lib.go delete mode 100644 server/repology/scheduler.go delete mode 100644 server/types/pac/parser/parse_test.go delete mode 100644 server/types/pac/parser/scheduler.go diff --git a/server/.gitignore b/server/.gitignore index dc5734a9..7a976121 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -4,3 +4,4 @@ tmp/ dist/ programs/ main +repology_cache.json diff --git a/server/Makefile b/server/Makefile index c80a59b7..a48a9cfe 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,69 +1,45 @@ LDFLAGS= \ -w \ -s \ - -X 'pacstall.dev/webserver/config/build.UpdateInterval=900' \ - -X 'pacstall.dev/webserver/config/build.TempDir=./tmp' \ - -X 'pacstall.dev/webserver/config/build.MaxOpenFiles=100' \ - -X 'pacstall.dev/webserver/config/build.Port=3300' \ - -X 'pacstall.dev/webserver/config/build.PublicDir=./public' \ - -X 'pacstall.dev/webserver/config/build.Production=true' \ - -X 'pacstall.dev/webserver/config/build.GitURL=https://github.com/pacstall/pacstall-programs.git' \ - -X 'pacstall.dev/webserver/config/build.GitClonePath=./programs' \ - -X 'pacstall.dev/webserver/config/build.Version=${VERSION}' + -X 'pacstall.dev/webserver/pkg/common/config/build.Production=true' \ + -X 'pacstall.dev/webserver/pkg/common/config/build.Version=${VERSION}' -all: dist/webserver +all: dist/pacnexus dist/pacsight test: - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ GO_ENV=test go test -v types/pac/parser/parse_test.go test_internal: - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ GO_ENV=test go test -v types/pac/parser/pacsh/internal/git_version_test.go -run: - (cd .. && docker compose up -d mariadb) +run-database: + @(cd .. && docker compose up -d mariadb) - PACSTALL_DATABASE_HOST=localhost \ - PACSTALL_DATABASE_PORT=3306 \ - PACSTALL_DATABASE_USER=root \ - PACSTALL_DATABASE_PASSWORD=changeme \ - PACSTALL_DATABASE_NAME=pacstall \ - PACSTALL_DISCORD_ENABLED=false \ - PACSTALL_DISCORD_TOKEN="" \ - PACSTALL_DISCORD_CHANNEL_ID="" \ - PACSTALL_DISCORD_TAGS="" \ - PACSTALL_MATOMO_ENABLED="false" \ - PACSTALL_REPOLOGY_ENABLED="false" \ - PACSTALL_PROGRAMS_GIT_BRANCH="master" \ - go run bin/webserver/main.go +run-pacnexus: run-database + @go run cmd/pacnexus/main.go + +run-pacsight: + @go run cmd/pacsight/main.go + + +run: + @trap 'kill 0' SIGINT; \ + $(MAKE) run-pacsight | sed 's/^/[pacsight] /' & \ + $(MAKE) run-pacnexus | sed 's/^/[pacnexus] /' & \ + wait + @echo "Both executables completed or interrupted." + +dist/pacnexus: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) + CGO_ENABLED=0 go build -o dist/pacnexus -ldflags "${LDFLAGS}" cmd/pacnexus/main.go + +dist/pacsight: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) + CGO_ENABLED=0 go build -o dist/pacsight -ldflags "${LDFLAGS}" cmd/pacsight/main.go -dist/webserver: $(shell find . -not \( -path ./tmp -prune \) -not \( -path ./dist -prune \) -type f) - CGO_ENABLED=0 go build -o dist/webserver -ldflags "${LDFLAGS}" bin/webserver/main.go clean: - [ -d ./dist ] && rm -r dist || : + @[ -d ./dist ] && rm -rf dist || : + @[ -d ./programs ] && rm -rf programs || : + @[ -d ./tmp ] && rm -rf tmp || : + @echo "Cleaned up." fmt: go fmt ./... diff --git a/server/bin/webserver/main.go b/server/bin/webserver/main.go deleted file mode 100644 index 97008335..00000000 --- a/server/bin/webserver/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/fatih/color" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/repology" - "pacstall.dev/webserver/server" - ps_api "pacstall.dev/webserver/server/api/pacscripts" - repology_api "pacstall.dev/webserver/server/api/repology" - urlshortener "pacstall.dev/webserver/server/api/url_shortener" - pac_ssr "pacstall.dev/webserver/server/ssr/pacscript" - "pacstall.dev/webserver/types/pac/parser" -) - -func printLogo() { - logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() - fmt.Println(logoColor(` - ____ __ ____ - / __ \____ ___________/ /_____ _/ / / - / /_/ / __ '/ ___/ ___/ __/ __ '/ / / - / ____/ /_/ / /__(__ ) /_/ /_/ / / / -/_/ _\__,_/\___/____/\__/\__,_/_/_/ -| | / /__ / /_ / ___/___ ______ _____ _____ -| | /| / / _ \/ __ \\__ \/ _ \/ ___/ | / / _ \/ ___/ -| |/ |/ / __/ /_/ /__/ / __/ / | |/ / __/ / -|__/|__/\___/_.___/____/\___/_/ |___/\___/_/ - coded by saenai255, owned by Pacstall Org - `)) -} - -func setupRequests() { - router := server.Router() - - /* Packages */ - pac_ssr.EnableSSR() - router.HandleFunc("/api/repology", repology_api.GetRepologyPackageListHandle).Methods("GET") - router.HandleFunc("/api/packages", ps_api.GetPacscriptListHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}", ps_api.GetPacscriptHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}/requiredBy", ps_api.GetPacscriptRequiredByHandle).Methods("GET") - router.HandleFunc("/api/packages/{name}/dependencies", ps_api.GetPacscriptDependenciesHandle).Methods("GET") - - /* Shortened Links - Must be last as it functions as a catch-all trap */ - router.HandleFunc("/q/{linkId}", urlshortener.GetShortenedLinkRedirectHandle).Methods("GET") -} - -func main() { - if config.Production { - log.SetLogLevel(log.Level.Info) - } else { - log.SetLogLevel(log.Level.Debug) - } - - startedAt := time.Now() - - printLogo() - - setupRequests() - log.Info("registered http requests") - - log.Info("attempting to start TCP listener") - - server.OnServerOnline(func() { - log.NotifyCustom("🚀 Startup 🧑‍🚀", "successfully started up.") - log.Info("server is now online on port %v.\n", config.Port) - - log.Info("booted in %v\n", color.GreenString("%v", time.Since(startedAt))) - - parser.ScheduleRefresh(config.UpdateInterval) - log.Info("scheduled pacscripts to auto-refresh every %v", config.UpdateInterval) - - if config.Repology.Enabled { - repology.ScheduleRefresh(config.RepologyUpdateInterval) - log.Info("scheduled repology to auto-refresh every %v", config.RepologyUpdateInterval) - } else { - log.Warn("repository repology is disabled") - } - }) - - server.Listen(config.Port) -} diff --git a/server/cmd/pacnexus/main.go b/server/cmd/pacnexus/main.go new file mode 100644 index 00000000..08e87645 --- /dev/null +++ b/server/cmd/pacnexus/main.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + "time" + + "github.com/fatih/color" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/server" + ps_api "pacstall.dev/webserver/internal/pacnexus/server/api/pacscripts" + repology_api "pacstall.dev/webserver/internal/pacnexus/server/api/repology" + urlshortener "pacstall.dev/webserver/internal/pacnexus/server/api/url_shortener" + pac_ssr "pacstall.dev/webserver/internal/pacnexus/server/ssr/pacscript" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser" + globalConfig "pacstall.dev/webserver/pkg/common/config" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func printLogo() { + logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() + fmt.Println(logoColor(` +88888b. 8888b. .d8888b 88888b. .d88b. 888 888 888 888 .d8888b +888 "88b "88b d88P" 888 "88b d8P Y8b Y8bd8P' 888 888 88K +888 888 .d888888 888 888 888 88888888 X88K 888 888 "Y8888b. +888 d88P 888 888 Y88b. 888 888 Y8b. .d8""8b. Y88b 888 X88 +88888P" "Y888888 "Y8888P 888 888 "Y8888 888 888 "Y88888 88888P' +888 +888 +888 + + coded by saenai255, owned by Pacstall Org + `)) +} + +func setupRequests() { + router := server.Router() + + /* Packages */ + pac_ssr.EnableSSR() + router.HandleFunc("/api/repology", repology_api.GetRepologyPackageListHandle).Methods("GET") + router.HandleFunc("/api/packages", ps_api.GetPacscriptListHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}", ps_api.GetPacscriptHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}/requiredBy", ps_api.GetPacscriptRequiredByHandle).Methods("GET") + router.HandleFunc("/api/packages/{name}/dependencies", ps_api.GetPacscriptDependenciesHandle).Methods("GET") + + /* Shortened Links - Must be last as it functions as a catch-all trap */ + router.HandleFunc("/q/{linkId}", urlshortener.GetShortenedLinkRedirectHandle).Methods("GET") +} + +func main() { + config.Init() + + if globalConfig.Production { + log.SetLogLevel(log.Level.Info) + } else { + log.SetLogLevel(log.Level.Debug) + } + + startedAt := time.Now() + + printLogo() + + setupRequests() + log.Info("registered http requests") + + pacsightRpc, err := pacsight.NewPacsightRpcService("localhost", 8080) + if err != nil { + log.Error("failed to create pacsight rpc service: %+v", err) + return + } + log.Info("connected to pacsight rpc service") + + log.Info("attempting to start tcp listener") + + server.OnServerOnline(func() { + log.Info("server is now online on port %v.\n", config.PacNexus.Port) + + log.Info("booted in %v\n", color.GreenString("%v", time.Since(startedAt))) + + parser.ScheduleRefresh(config.PacstallPrograms.UpdateInterval, pacsightRpc) + log.Info("scheduled pacscripts to auto-refresh every %v", config.PacstallPrograms.UpdateInterval) + }) + + server.Listen(config.PacNexus.Port) +} diff --git a/server/cmd/pacsight/main.go b/server/cmd/pacsight/main.go new file mode 100644 index 00000000..6500d937 --- /dev/null +++ b/server/cmd/pacsight/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "net" + "net/http" + + "github.com/fatih/color" + "pacstall.dev/webserver/internal/pacsight/config" + "pacstall.dev/webserver/internal/pacsight/repology" + "pacstall.dev/webserver/internal/pacsight/rpcall" + "pacstall.dev/webserver/pkg/common/log" +) + +func printLogo() { + logoColor := color.New(color.FgHiMagenta, color.Bold).SprintFunc() + fmt.Println(logoColor(` + d8b 888 888 + Y8P 888 888 + 888 888 +88888b. 8888b. .d8888b .d8888b 888 .d88b. 88888b. 888888 +888 "88b "88b d88P" 88K 888 d88P"88b 888 "88b 888 +888 888 .d888888 888 "Y8888b. 888 888 888 888 888 888 +888 d88P 888 888 Y88b. X88 888 Y88b 888 888 888 Y88b. +88888P" "Y888888 "Y8888P 88888P' 888 "Y88888 888 888 "Y888 +888 888 +888 Y8b d88P +888 "Y88P" + + coded by saenai255, owned by Pacstall Org + `)) +} + +func main() { + config.Init() + printLogo() + + log.Info("booting up pacsight") + + go repology.ScheduleRefresh(config.Repology.RepologyUpdateInterval) + rpcall.RegisterService() + + port := fmt.Sprintf(":%v", config.PacSight.Port) + listener, err := net.Listen("tcp", port) + if err != nil { + log.Fatal("failed to start server: %+v", err) + } + + log.Info("server started on port " + port) + + if err := http.Serve(listener, nil); err != nil { + log.Fatal("failed to serve: %+v", err) + } + +} diff --git a/server/config/build/vars.go b/server/config/build/vars.go deleted file mode 100644 index 1b1fc378..00000000 --- a/server/config/build/vars.go +++ /dev/null @@ -1,14 +0,0 @@ -package build - -var Production = "false" - -var UpdateInterval = "900" -var RepologyUpdateInterval = "43200" -var TempDir = "./tmp" -var MaxOpenFiles = "100" -var GitURL = "https://github.com/pacstall/pacstall-programs.git" -var GitClonePath = "./programs" - -var Port = "3300" -var PublicDir = "../client/dist" -var Version = "development" diff --git a/server/config/env.go b/server/config/env.go deleted file mode 100644 index a70e0865..00000000 --- a/server/config/env.go +++ /dev/null @@ -1,49 +0,0 @@ -package config - -// Configuration for the discord integration -var Discord = struct { - Token string - ChannelID string - Enabled bool - Tags string -}{ - Token: getEnvString("PACSTALL_DISCORD_TOKEN"), - ChannelID: getEnvString("PACSTALL_DISCORD_CHANNEL_ID"), - Enabled: getEnvBool("PACSTALL_DISCORD_ENABLED"), - Tags: getEnvString("PACSTALL_DISCORD_TAGS"), -} - -var PacstallPrograms = struct { - Branch string -}{ - Branch: getEnvString("PACSTALL_PROGRAMS_GIT_BRANCH"), -} - -// Configuration for the database -var Database = struct { - Host string - Port int - User string - Password string - Name string -}{ - Host: getEnvString("PACSTALL_DATABASE_HOST"), - Port: getEnvInt("PACSTALL_DATABASE_PORT"), - User: getEnvString("PACSTALL_DATABASE_USER"), - Password: getEnvString("PACSTALL_DATABASE_PASSWORD"), - Name: getEnvString("PACSTALL_DATABASE_NAME"), -} - -// Configuration for the Matomo API -var Matomo = struct { - Enabled bool -}{ - Enabled: getEnvBool("PACSTALL_MATOMO_ENABLED"), -} - -// Configuration for the Repology API -var Repology = struct { - Enabled bool -}{ - Enabled: getEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", true), -} diff --git a/server/config/vars.go b/server/config/vars.go deleted file mode 100644 index d41d7f64..00000000 --- a/server/config/vars.go +++ /dev/null @@ -1,21 +0,0 @@ -package config - -import ( - "time" - - "pacstall.dev/webserver/config/build" -) - -var Production = toBool(build.Production) - -var UpdateInterval = time.Duration(toInt(build.UpdateInterval)) * time.Second -var RepologyUpdateInterval = time.Duration(toInt(build.RepologyUpdateInterval)) * time.Second -var TempDir = build.TempDir -var MaxOpenFiles = toInt(build.MaxOpenFiles) -var GitURL = build.GitURL -var GitClonePath = build.GitClonePath - -var Port = toInt(build.Port) -var PublicDir = build.PublicDir - -var Version = build.Version diff --git a/server/internal/pacnexus/config/env.go b/server/internal/pacnexus/config/env.go new file mode 100644 index 00000000..7228c610 --- /dev/null +++ b/server/internal/pacnexus/config/env.go @@ -0,0 +1,108 @@ +package config + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/env" +) + +var PacSight = struct { + Port int +}{} + +func initPacSightEnv() { + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) +} + +var PacNexus = struct { + Port int + PublicDir string +}{} + +func initPacNexusEnv() { + PacNexus.Port = env.GetEnvIntOrDefault("PACSTALL_PACNEXUS_PORT", 3300) + PacNexus.PublicDir = env.GetEnvStringOrDefault("PACSTALL_PACNEXUS_PUBLIC_DIR", "./public") +} + +// Configuration for the discord integration +var Discord = struct { + Token string + ChannelID string + Enabled bool + Tags string +}{} + +func initDiscordEnv() { + Discord.Enabled = env.GetEnvBoolOrDefault("PACSTALL_DISCORD_ENABLED", false) + + if !Discord.Enabled { + return + } + + Discord.Token = env.GetEnvString("PACSTALL_DISCORD_TOKEN") + Discord.ChannelID = env.GetEnvString("PACSTALL_DISCORD_CHANNEL_ID") + Discord.Tags = env.GetEnvString("PACSTALL_DISCORD_TAGS") +} + +var PacstallPrograms = struct { + Branch string + UpdateInterval time.Duration + TempDir string + MaxOpenFiles int + GitURL string + GitClonePath string +}{} + +func initPacstallProgramsEnv() { + PacstallPrograms.Branch = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_BRANCH", "master") + PacstallPrograms.UpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_PROGRAMS_UPDATE_INTERVAL", 15*60)) * time.Second + PacstallPrograms.TempDir = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_TEMP_DIR", "./tmp") + PacstallPrograms.MaxOpenFiles = env.GetEnvIntOrDefault("PACSTALL_PROGRAMS_MAX_OPEN_FILES", 100) + PacstallPrograms.GitURL = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_URL", "https://github.com/pacstall/pacstall-programs.git") + PacstallPrograms.GitClonePath = env.GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_CLONE_PATH", "./programs") +} + +// Configuration for the database +var Database = struct { + Host string + Port int + User string + Password string + Name string +}{} + +func initDatabaseEnv() { + Database.Host = env.GetEnvStringOrDefault("PACSTALL_DATABASE_HOST", "localhost") + Database.Port = env.GetEnvIntOrDefault("PACSTALL_DATABASE_PORT", 3306) + Database.User = env.GetEnvStringOrDefault("PACSTALL_DATABASE_USER", "root") + Database.Password = env.GetEnvStringOrDefault("PACSTALL_DATABASE_PASSWORD", "changeme") + Database.Name = env.GetEnvStringOrDefault("PACSTALL_DATABASE_NAME", "pacstall") +} + +// Configuration for the Matomo API +var Matomo = struct { + Enabled bool +}{} + +func initMatomoEnv() { + Matomo.Enabled = env.GetEnvBoolOrDefault("PACSTALL_MATOMO_ENABLED", false) +} + +// Configuration for the Repology API +var Repology = struct { + Enabled bool +}{} + +func initRepologyEnv() { + Repology.Enabled = env.GetEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", false) +} + +func Init() { + initPacNexusEnv() + initPacSightEnv() + initDatabaseEnv() + initDiscordEnv() + initMatomoEnv() + initPacstallProgramsEnv() + initRepologyEnv() +} diff --git a/server/consts/consts.go b/server/internal/pacnexus/consts/consts.go similarity index 100% rename from server/consts/consts.go rename to server/internal/pacnexus/consts/consts.go diff --git a/server/model/connection.go b/server/internal/pacnexus/model/connection.go similarity index 60% rename from server/model/connection.go rename to server/internal/pacnexus/model/connection.go index 614d7f55..61e60dd7 100644 --- a/server/model/connection.go +++ b/server/internal/pacnexus/model/connection.go @@ -7,18 +7,21 @@ import ( "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/pkg/common/log" ) var database *gorm.DB = nil -var connectionString = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", - config.Database.User, - config.Database.Password, - config.Database.Host, - config.Database.Port, - config.Database.Name, -) + +func getConnectionString() string { + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", + config.Database.User, + config.Database.Password, + config.Database.Host, + config.Database.Port, + config.Database.Name, + ) +} func Instance() *gorm.DB { if database != nil { @@ -26,7 +29,7 @@ func Instance() *gorm.DB { } err := retry(5, func() (err error) { - database, err = gorm.Open(mysql.Open(connectionString), &gorm.Config{ + database, err = gorm.Open(mysql.Open(getConnectionString()), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) @@ -34,7 +37,7 @@ func Instance() *gorm.DB { }) if err != nil { - panic(fmt.Sprintf("failed to connect database: %v", err)) + panic(fmt.Sprintf("failed to connect database: %+v", err)) } log.Info("connected to database.") diff --git a/server/model/repology_project.go b/server/internal/pacnexus/model/repology_project.go similarity index 100% rename from server/model/repology_project.go rename to server/internal/pacnexus/model/repology_project.go diff --git a/server/model/repology_project_provider.go b/server/internal/pacnexus/model/repology_project_provider.go similarity index 100% rename from server/model/repology_project_provider.go rename to server/internal/pacnexus/model/repology_project_provider.go diff --git a/server/model/shortened_link.go b/server/internal/pacnexus/model/shortened_link.go similarity index 100% rename from server/model/shortened_link.go rename to server/internal/pacnexus/model/shortened_link.go diff --git a/server/server/api/pacscripts/dependencies.go b/server/internal/pacnexus/server/api/pacscripts/dependencies.go similarity index 85% rename from server/server/api/pacscripts/dependencies.go rename to server/internal/pacnexus/server/api/pacscripts/dependencies.go index 07ddc522..14221f28 100644 --- a/server/server/api/pacscripts/dependencies.go +++ b/server/internal/pacnexus/server/api/pacscripts/dependencies.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) type pacscriptDependencies struct { diff --git a/server/server/api/pacscripts/package.go b/server/internal/pacnexus/server/api/pacscripts/package.go similarity index 73% rename from server/server/api/pacscripts/package.go rename to server/internal/pacnexus/server/api/pacscripts/package.go index c3fbfad8..09dee1e6 100644 --- a/server/server/api/pacscripts/package.go +++ b/server/internal/pacnexus/server/api/pacscripts/package.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetPacscriptHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/pacscripts/package_list.go b/server/internal/pacnexus/server/api/pacscripts/package_list.go similarity index 89% rename from server/server/api/pacscripts/package_list.go rename to server/internal/pacnexus/server/api/pacscripts/package_list.go index c35c1e41..550dcba9 100644 --- a/server/server/api/pacscripts/package_list.go +++ b/server/internal/pacnexus/server/api/pacscripts/package_list.go @@ -5,11 +5,11 @@ import ( "math" "net/http" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/server/query" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" - "pacstall.dev/webserver/types/pac/parser" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/server/query" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser" ) type packageListPage struct { diff --git a/server/server/api/pacscripts/required_by.go b/server/internal/pacnexus/server/api/pacscripts/required_by.go similarity index 79% rename from server/server/api/pacscripts/required_by.go rename to server/internal/pacnexus/server/api/pacscripts/required_by.go index ccf66a52..06c5c7da 100644 --- a/server/server/api/pacscripts/required_by.go +++ b/server/internal/pacnexus/server/api/pacscripts/required_by.go @@ -5,10 +5,10 @@ import ( "net/http" "github.com/gorilla/mux" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetPacscriptRequiredByHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/repology/repology.go b/server/internal/pacnexus/server/api/repology/repology.go similarity index 74% rename from server/server/api/repology/repology.go rename to server/internal/pacnexus/server/api/repology/repology.go index 82b8415e..ce72f761 100644 --- a/server/server/api/repology/repology.go +++ b/server/internal/pacnexus/server/api/repology/repology.go @@ -4,10 +4,10 @@ import ( "fmt" "net/http" - "pacstall.dev/webserver/server" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/server" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func GetRepologyPackageListHandle(w http.ResponseWriter, req *http.Request) { diff --git a/server/server/api/repology/types.go b/server/internal/pacnexus/server/api/repology/types.go similarity index 97% rename from server/server/api/repology/types.go rename to server/internal/pacnexus/server/api/repology/types.go index de81339c..3c5a4d6f 100644 --- a/server/server/api/repology/types.go +++ b/server/internal/pacnexus/server/api/repology/types.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" ) type maintainerDetails struct { diff --git a/server/server/api/url_shortener/url_shortener.go b/server/internal/pacnexus/server/api/url_shortener/url_shortener.go similarity index 83% rename from server/server/api/url_shortener/url_shortener.go rename to server/internal/pacnexus/server/api/url_shortener/url_shortener.go index a31576b1..06de3bdf 100644 --- a/server/server/api/url_shortener/url_shortener.go +++ b/server/internal/pacnexus/server/api/url_shortener/url_shortener.go @@ -7,12 +7,11 @@ import ( "github.com/gorilla/mux" "github.com/treelightsoftware/go-matomo" "gorm.io/gorm" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/model" + "pacstall.dev/webserver/pkg/common/log" ) -var db = model.Instance() var incrementVisitsExpression = gorm.Expr(model.ShortenedLinkColumns.Visits+" + ?", 1) var pathParams = struct { @@ -40,7 +39,7 @@ func GetShortenedLinkRedirectHandle(w http.ResponseWriter, req *http.Request) { } var shortenedLink model.ShortenedLink - if result := db.Where(model.ShortenedLink{LinkId: linkId}).First(&shortenedLink); result.Error != nil { + if result := model.Instance().Where(model.ShortenedLink{LinkId: linkId}).First(&shortenedLink); result.Error != nil { w.WriteHeader(404) return } @@ -51,7 +50,7 @@ func GetShortenedLinkRedirectHandle(w http.ResponseWriter, req *http.Request) { return } - db.Model(&shortenedLink).Update(model.ShortenedLinkColumns.Visits, incrementVisitsExpression) + model.Instance().Model(&shortenedLink).Update(model.ShortenedLinkColumns.Visits, incrementVisitsExpression) if config.Matomo.Enabled { pingMatomoTracker(req.RemoteAddr, req.UserAgent(), req.Referer(), linkId) } diff --git a/server/server/headers.go b/server/internal/pacnexus/server/headers.go similarity index 95% rename from server/server/headers.go rename to server/internal/pacnexus/server/headers.go index a1a8b105..60a3d598 100644 --- a/server/server/headers.go +++ b/server/internal/pacnexus/server/headers.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - "pacstall.dev/webserver/config" + "pacstall.dev/webserver/pkg/common/config" ) type AlreadyResponded = bool diff --git a/server/server/health_check.go b/server/internal/pacnexus/server/health_check.go similarity index 95% rename from server/server/health_check.go rename to server/internal/pacnexus/server/health_check.go index 669e6e73..702b787d 100644 --- a/server/server/health_check.go +++ b/server/internal/pacnexus/server/health_check.go @@ -5,7 +5,7 @@ import ( "net/http" "time" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func registerHealthCheck() { diff --git a/server/server/query/main.go b/server/internal/pacnexus/server/query/main.go similarity index 100% rename from server/server/query/main.go rename to server/internal/pacnexus/server/query/main.go diff --git a/server/server/sitemap.go b/server/internal/pacnexus/server/sitemap.go similarity index 96% rename from server/server/sitemap.go rename to server/internal/pacnexus/server/sitemap.go index 74321039..21adac94 100644 --- a/server/server/sitemap.go +++ b/server/internal/pacnexus/server/sitemap.go @@ -4,7 +4,7 @@ import ( "fmt" "net/http" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" ) type SitemapEntry struct { diff --git a/server/server/spa.go b/server/internal/pacnexus/server/spa.go similarity index 96% rename from server/server/spa.go rename to server/internal/pacnexus/server/spa.go index 35e6a9d4..75baa352 100644 --- a/server/server/spa.go +++ b/server/internal/pacnexus/server/spa.go @@ -8,7 +8,7 @@ import ( text "text/template" - "pacstall.dev/webserver/server/ssr" + "pacstall.dev/webserver/internal/pacnexus/server/ssr" ) type spaHandler struct { diff --git a/server/server/ssr/pacscript/main.go b/server/internal/pacnexus/server/ssr/pacscript/main.go similarity index 100% rename from server/server/ssr/pacscript/main.go rename to server/internal/pacnexus/server/ssr/pacscript/main.go diff --git a/server/server/ssr/pacscript/package.go b/server/internal/pacnexus/server/ssr/pacscript/package.go similarity index 84% rename from server/server/ssr/pacscript/package.go rename to server/internal/pacnexus/server/ssr/pacscript/package.go index 4edeafb7..02414077 100644 --- a/server/server/ssr/pacscript/package.go +++ b/server/internal/pacnexus/server/ssr/pacscript/package.go @@ -5,11 +5,11 @@ import ( "regexp" "strings" - "pacstall.dev/webserver/consts" - r "pacstall.dev/webserver/server/ssr" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/consts" + r "pacstall.dev/webserver/internal/pacnexus/server/ssr" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/pkg/common/array" ) func registerPacscriptSSRData() { diff --git a/server/server/ssr/pacscript/packages.go b/server/internal/pacnexus/server/ssr/pacscript/packages.go similarity index 88% rename from server/server/ssr/pacscript/packages.go rename to server/internal/pacnexus/server/ssr/pacscript/packages.go index 8ba86c15..5e4f9da1 100644 --- a/server/server/ssr/pacscript/packages.go +++ b/server/internal/pacnexus/server/ssr/pacscript/packages.go @@ -3,7 +3,7 @@ package ssr import ( "regexp" - r "pacstall.dev/webserver/server/ssr" + r "pacstall.dev/webserver/internal/pacnexus/server/ssr" ) func registerPacscriptListSSRData() { diff --git a/server/server/ssr/templates.go b/server/internal/pacnexus/server/ssr/templates.go similarity index 100% rename from server/server/ssr/templates.go rename to server/internal/pacnexus/server/ssr/templates.go diff --git a/server/server/webserver.go b/server/internal/pacnexus/server/webserver.go similarity index 80% rename from server/server/webserver.go rename to server/internal/pacnexus/server/webserver.go index 1602491a..a1e6a177 100644 --- a/server/server/webserver.go +++ b/server/internal/pacnexus/server/webserver.go @@ -10,8 +10,9 @@ import ( "time" "github.com/gorilla/mux" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/internal/pacnexus/config" + globalConfig "pacstall.dev/webserver/pkg/common/config" + "pacstall.dev/webserver/pkg/common/log" ) var router mux.Router = *mux.NewRouter() @@ -27,7 +28,7 @@ func Listen(port int) { Router().Use(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Pacstall-Version", config.Version) + w.Header().Add("X-Pacstall-Version", globalConfig.Version) if strings.Contains(r.URL.Path, "/api") { w.Header().Add("Content-Type", "application/json") @@ -38,10 +39,10 @@ func Listen(port int) { go triggerServerOnline(port) - if config.Production { - path, err := filepath.Abs(config.PublicDir) + if globalConfig.Production { + path, err := filepath.Abs(config.PacNexus.PublicDir) if err != nil { - log.Fatal("failed to find client public dir at path '%s'. err: %+v", config.PublicDir, err) + log.Fatal("failed to find client public dir at path '%s'. err: %+v", config.PacNexus.PublicDir, err) } Router().PathPrefix("/").Handler(spaHandler{staticPath: path}) diff --git a/server/types/pac/pacstore/store.go b/server/internal/pacnexus/types/pac/pacstore/store.go similarity index 88% rename from server/types/pac/pacstore/store.go rename to server/internal/pacnexus/types/pac/pacstore/store.go index e38bf0fe..e331e9e9 100644 --- a/server/types/pac/pacstore/store.go +++ b/server/internal/pacnexus/types/pac/pacstore/store.go @@ -3,8 +3,8 @@ package pacstore import ( "time" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) var lastModified time.Time diff --git a/server/types/pac/parser/git/lib.go b/server/internal/pacnexus/types/pac/parser/git/lib.go similarity index 98% rename from server/types/pac/parser/git/lib.go rename to server/internal/pacnexus/types/pac/parser/git/lib.go index 0852cac7..92d62ec8 100644 --- a/server/types/pac/parser/git/lib.go +++ b/server/internal/pacnexus/types/pac/parser/git/lib.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func hardResetAndPull(path, branch string) error { diff --git a/server/types/pac/parser/last_updated.go b/server/internal/pacnexus/types/pac/parser/last_updated.go similarity index 87% rename from server/types/pac/parser/last_updated.go rename to server/internal/pacnexus/types/pac/parser/last_updated.go index adf1bb8d..f29314d7 100644 --- a/server/types/pac/parser/last_updated.go +++ b/server/internal/pacnexus/types/pac/parser/last_updated.go @@ -9,11 +9,11 @@ import ( "time" "github.com/joomcode/errorx" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/log" ) type packageLastUpdatedTuple struct { @@ -27,7 +27,7 @@ func getPackageLastUpdatedTuples() ([]packageLastUpdatedTuple, error) { return nil, errorx.Decorate(err, "failed to get absolute path to wording directory") } - programsPath := path.Join(wordingDirectoryAbsolute, config.GitClonePath) + programsPath := path.Join(wordingDirectoryAbsolute, config.PacstallPrograms.GitClonePath) script := fmt.Sprintf(` cd %v for i in ./packages/*/*.pacscript; do echo $i; git log -1 --pretty=\"%%at\" $i; done diff --git a/server/types/pac/parser/pacscript.go b/server/internal/pacnexus/types/pac/parser/pacscript.go similarity index 84% rename from server/types/pac/parser/pacscript.go rename to server/internal/pacnexus/types/pac/parser/pacscript.go index 82c3032b..2ee7efb5 100644 --- a/server/types/pac/parser/pacscript.go +++ b/server/internal/pacnexus/types/pac/parser/pacscript.go @@ -1,8 +1,8 @@ package parser import ( - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) func computeRequiredBy(script *pac.Script, scripts []*pac.Script) { diff --git a/server/types/pac/parser/pacscript_list.go b/server/internal/pacnexus/types/pac/parser/pacscript_list.go similarity index 76% rename from server/types/pac/parser/pacscript_list.go rename to server/internal/pacnexus/types/pac/parser/pacscript_list.go index d109ec37..945bf265 100644 --- a/server/types/pac/parser/pacscript_list.go +++ b/server/internal/pacnexus/types/pac/parser/pacscript_list.go @@ -1,6 +1,6 @@ package parser -import "pacstall.dev/webserver/types/pac" +import "pacstall.dev/webserver/internal/pacnexus/types/pac" type PacscriptListWrapper []*pac.Script diff --git a/server/types/pac/parser/pacsh/exec_sh.go b/server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go similarity index 94% rename from server/types/pac/parser/pacsh/exec_sh.go rename to server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go index e268df76..3b9482ac 100644 --- a/server/types/pac/parser/pacsh/exec_sh.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/exec_sh.go @@ -4,7 +4,7 @@ import ( "os" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) var removeFile = os.Remove diff --git a/server/types/pac/parser/pacsh/git_version.go b/server/internal/pacnexus/types/pac/parser/pacsh/git_version.go similarity index 76% rename from server/types/pac/parser/pacsh/git_version.go rename to server/internal/pacnexus/types/pac/parser/pacsh/git_version.go index c40d00f9..dc99e292 100644 --- a/server/types/pac/parser/pacsh/git_version.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/git_version.go @@ -2,9 +2,9 @@ package pacsh import ( "github.com/joomcode/errorx" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/pkg/common/array" ) func ApplyGitVersion(p *pac.Script) error { diff --git a/server/types/pac/parser/pacsh/internal/git_version.go b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go similarity index 96% rename from server/types/pac/parser/pacsh/internal/git_version.go rename to server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go index 6c80ee32..7c63f746 100644 --- a/server/types/pac/parser/pacsh/internal/git_version.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go @@ -6,8 +6,8 @@ import ( "time" "github.com/joomcode/errorx" - "pacstall.dev/webserver/types/pac/parser/git" - "pacstall.dev/webserver/types/pac/parser/parallelism/timeout" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/timeout" ) type GitSourceInfo struct { diff --git a/server/types/pac/parser/pacsh/internal/git_version_test.go b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go similarity index 97% rename from server/types/pac/parser/pacsh/internal/git_version_test.go rename to server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go index 6e3d7f2f..ca87ff16 100644 --- a/server/types/pac/parser/pacsh/internal/git_version_test.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version_test.go @@ -3,7 +3,7 @@ package internal_test import ( "testing" - "pacstall.dev/webserver/types/pac/parser/pacsh/internal" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh/internal" ) func assertGitSourceInfoEquals(t *testing.T, expected, actual internal.GitSourceInfo) { diff --git a/server/types/pac/parser/pacsh/parse_pac_output.go b/server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go similarity index 83% rename from server/types/pac/parser/pacsh/parse_pac_output.go rename to server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go index 21fd336f..197bd0cd 100644 --- a/server/types/pac/parser/pacsh/parse_pac_output.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/parse_pac_output.go @@ -2,7 +2,7 @@ package pacsh import ( "github.com/pacstall/go-srcinfo" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" ) func ParsePacOutput(data []byte) (*pac.Script, error) { diff --git a/server/types/pac/parser/pacsh/pretty-name.go b/server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go similarity index 86% rename from server/types/pac/parser/pacsh/pretty-name.go rename to server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go index d1e12030..a75414d7 100644 --- a/server/types/pac/parser/pacsh/pretty-name.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/pretty-name.go @@ -3,8 +3,8 @@ package pacsh import ( "strings" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/types" ) func getPrettyName(p *pac.Script) string { diff --git a/server/types/pac/parser/pacsh/temp_dir.go b/server/internal/pacnexus/types/pac/parser/pacsh/temp_dir.go similarity index 100% rename from server/types/pac/parser/pacsh/temp_dir.go rename to server/internal/pacnexus/types/pac/parser/pacsh/temp_dir.go diff --git a/server/types/pac/parser/pacsh/temp_exec.go b/server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go similarity index 96% rename from server/types/pac/parser/pacsh/temp_exec.go rename to server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go index b87028e6..ab404214 100644 --- a/server/types/pac/parser/pacsh/temp_exec.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/temp_exec.go @@ -7,7 +7,7 @@ import ( "path" "github.com/joomcode/errorx" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) var createFile = os.Create diff --git a/server/types/pac/parser/parallelism/batch/run.go b/server/internal/pacnexus/types/pac/parser/parallelism/batch/run.go similarity index 93% rename from server/types/pac/parser/parallelism/batch/run.go rename to server/internal/pacnexus/types/pac/parser/parallelism/batch/run.go index 95adb852..bf071c8a 100644 --- a/server/types/pac/parser/parallelism/batch/run.go +++ b/server/internal/pacnexus/types/pac/parser/parallelism/batch/run.go @@ -4,7 +4,7 @@ import ( "sync/atomic" "time" - "pacstall.dev/webserver/log" + "pacstall.dev/webserver/pkg/common/log" ) func Run[T any, E any](batchSize int, items []T, fn func(T) (E, error)) <-chan E { diff --git a/server/types/pac/parser/parallelism/batch/run_test.go b/server/internal/pacnexus/types/pac/parser/parallelism/batch/run_test.go similarity index 72% rename from server/types/pac/parser/parallelism/batch/run_test.go rename to server/internal/pacnexus/types/pac/parser/parallelism/batch/run_test.go index 7268e397..88d069ca 100644 --- a/server/types/pac/parser/parallelism/batch/run_test.go +++ b/server/internal/pacnexus/types/pac/parser/parallelism/batch/run_test.go @@ -4,9 +4,9 @@ import ( "testing" "time" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac/parser/parallelism/batch" - "pacstall.dev/webserver/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/batch" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/pkg/common/array" ) func Test_Batch_Run(t *testing.T) { diff --git a/server/types/pac/parser/parallelism/channels/exhaust.go b/server/internal/pacnexus/types/pac/parser/parallelism/channels/exhaust.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/exhaust.go rename to server/internal/pacnexus/types/pac/parser/parallelism/channels/exhaust.go diff --git a/server/types/pac/parser/parallelism/channels/to_slice.go b/server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/to_slice.go rename to server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice.go diff --git a/server/types/pac/parser/parallelism/channels/to_slice_test.go b/server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice_test.go similarity index 100% rename from server/types/pac/parser/parallelism/channels/to_slice_test.go rename to server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice_test.go diff --git a/server/types/pac/parser/parallelism/timeout/timeout.go b/server/internal/pacnexus/types/pac/parser/parallelism/timeout/timeout.go similarity index 100% rename from server/types/pac/parser/parallelism/timeout/timeout.go rename to server/internal/pacnexus/types/pac/parser/parallelism/timeout/timeout.go diff --git a/server/types/pac/parser/parse.go b/server/internal/pacnexus/types/pac/parser/parse.go similarity index 60% rename from server/types/pac/parser/parse.go rename to server/internal/pacnexus/types/pac/parser/parse.go index 26eca283..08535b75 100644 --- a/server/types/pac/parser/parse.go +++ b/server/internal/pacnexus/types/pac/parser/parse.go @@ -6,25 +6,25 @@ import ( "strings" "github.com/joomcode/errorx" - "pacstall.dev/webserver/config" - "pacstall.dev/webserver/consts" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/repology" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/pacstore" - "pacstall.dev/webserver/types/pac/parser/git" - "pacstall.dev/webserver/types/pac/parser/pacsh" - "pacstall.dev/webserver/types/pac/parser/parallelism/batch" - "pacstall.dev/webserver/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/internal/pacnexus/consts" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/batch" + "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/channels" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" + "pacstall.dev/webserver/pkg/common/types" ) const PACKAGE_LIST_FILE_NAME = "./packagelist" const MAX_GIT_VERSION_CONCURRENCY = 5 -func ParseAll() error { - if err := git.RefreshPrograms(config.GitClonePath, config.GitURL, config.PacstallPrograms.Branch); err != nil { +func ParseAll(pacsightRpc *pacsight.PacsightRpcService) error { + if err := git.RefreshPrograms(config.PacstallPrograms.GitClonePath, config.PacstallPrograms.GitURL, config.PacstallPrograms.Branch); err != nil { return errorx.Decorate(err, "could not update repository 'pacstall-programs'") } @@ -33,7 +33,7 @@ func ParseAll() error { return errorx.Decorate(err, "failed to parse packagelist") } - loadedPacscripts, err := parsePacscriptFiles(pkgList) + loadedPacscripts, err := parsePacscriptFiles(pkgList, pacsightRpc) if err != nil { return errorx.Decorate(err, "failed to parse pacscripts") } @@ -72,7 +72,7 @@ func ParseAll() error { } func readKnownPacscriptNames() ([]string, error) { - pkglistPath := path.Join(config.GitClonePath, PACKAGE_LIST_FILE_NAME) + pkglistPath := path.Join(config.PacstallPrograms.GitClonePath, PACKAGE_LIST_FILE_NAME) bytes, err := os.ReadFile(pkglistPath) if err != nil { return nil, err @@ -86,18 +86,26 @@ func readKnownPacscriptNames() ([]string, error) { return names, nil } -func parsePacscriptFiles(names []string) ([]*pac.Script, error) { - if err := pacsh.CreateTempDirectory(config.TempDir); err != nil { +func parsePacscriptFiles(names []string, pacsightRpc *pacsight.PacsightRpcService) ([]*pac.Script, error) { + if err := pacsh.CreateTempDirectory(config.PacstallPrograms.TempDir); err != nil { return nil, errorx.Decorate(err, "failed to create temporary directory") } log.Info("parsing pacscripts...") - outChan := batch.Run(int(config.MaxOpenFiles), names, func(pacName string) (*pac.Script, error) { - out, err := ParsePacscriptFile(config.GitClonePath, pacName) + outChan := batch.Run(int(config.PacstallPrograms.MaxOpenFiles), names, func(pacName string) (*pac.Script, error) { + out, err := ParsePacscriptFile(config.PacstallPrograms.GitClonePath, pacName) + + if config.Repology.Enabled && len(out.Repology) > 0 { + project, err := pacsightRpc.GetRepologyProject(out.Repology) + if err != nil { + log.Debug("failed to get repology project %v. Error: %+v", out.Repology, err) + } else { + if err := UpdateScriptVersion(project, out); err != nil { + log.Warn("failed to update script version %v with repology. Error: %+v", pacName, err) + } + + out.LatestVersion = &project.Version - if config.Repology.Enabled { - if err := repology.Sync(out); err != nil { - log.Debug("failed to sync %v with repology. Error: %+v", pacName, err) } } diff --git a/server/internal/pacnexus/types/pac/parser/scheduler.go b/server/internal/pacnexus/types/pac/parser/scheduler.go new file mode 100644 index 00000000..7d1af143 --- /dev/null +++ b/server/internal/pacnexus/types/pac/parser/scheduler.go @@ -0,0 +1,28 @@ +package parser + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func ScheduleRefresh(every time.Duration, pacsightRpc *pacsight.PacsightRpcService) { + go refresh(every, pacsightRpc) +} + +func refresh(every time.Duration, pacsightRpc *pacsight.PacsightRpcService) { + for { + err := ParseAll(pacsightRpc) + if err != nil { + log.Error("parse error: %+v", err) + + retryIn := time.Second * 30 + log.Info("retrying in %v", retryIn) + time.Sleep(retryIn) + continue + } + + time.Sleep(every) + } +} diff --git a/server/types/pac/parser/search.go b/server/internal/pacnexus/types/pac/parser/search.go similarity index 96% rename from server/types/pac/parser/search.go rename to server/internal/pacnexus/types/pac/parser/search.go index f1b99407..f5f7595e 100644 --- a/server/types/pac/parser/search.go +++ b/server/internal/pacnexus/types/pac/parser/search.go @@ -3,8 +3,8 @@ package parser import ( "strings" - "pacstall.dev/webserver/types/array" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/array" ) const DEFAULT = "default" diff --git a/server/repology/sync.go b/server/internal/pacnexus/types/pac/parser/sync.go similarity index 89% rename from server/repology/sync.go rename to server/internal/pacnexus/types/pac/parser/sync.go index c1bd83c8..c4aceec0 100644 --- a/server/repology/sync.go +++ b/server/internal/pacnexus/types/pac/parser/sync.go @@ -1,11 +1,11 @@ -package repology +package parser import ( "strings" "github.com/hashicorp/go-version" - "pacstall.dev/webserver/model" - "pacstall.dev/webserver/types/pac" + "pacstall.dev/webserver/internal/pacnexus/types/pac" + "pacstall.dev/webserver/pkg/common/types" ) func compareNonStandardVersion(current, latest string) pac.UpdateStatusValue { @@ -21,7 +21,7 @@ func compareNonStandardVersion(current, latest string) pac.UpdateStatusValue { return pac.UpdateStatus.Major } -func updateScriptVersion(project model.RepologyProjectProvider, script *pac.Script) (err error) { +func UpdateScriptVersion(project types.RepologyApiProject, script *pac.Script) (err error) { script.LatestVersion = &project.Version if *script.LatestVersion == script.Version { @@ -80,7 +80,7 @@ func updateScriptVersion(project model.RepologyProjectProvider, script *pac.Scri return } -func versionCompare(current, latest string) pac.UpdateStatusValue { +func GetUpdateStatus(current, latest string) pac.UpdateStatusValue { min := func(a, b int) int { if a < b { return a diff --git a/server/types/pac/script.go b/server/internal/pacnexus/types/pac/script.go similarity index 98% rename from server/types/pac/script.go rename to server/internal/pacnexus/types/pac/script.go index 96b43298..b7272974 100644 --- a/server/types/pac/script.go +++ b/server/internal/pacnexus/types/pac/script.go @@ -5,8 +5,8 @@ import ( "time" "github.com/pacstall/go-srcinfo" - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/array" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/types" ) type updateStatus struct { diff --git a/server/internal/pacsight/config/env.go b/server/internal/pacsight/config/env.go new file mode 100644 index 00000000..ebd0ced1 --- /dev/null +++ b/server/internal/pacsight/config/env.go @@ -0,0 +1,28 @@ +package config + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/env" +) + +var PacSight = struct { + Port int +}{} + +func initPacSightEnv() { + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) +} + +var Repology = struct { + RepologyUpdateInterval time.Duration +}{} + +func initRepologyEnv() { + Repology.RepologyUpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second +} + +func Init() { + initPacSightEnv() + initRepologyEnv() +} diff --git a/server/internal/pacsight/repology/exporter.go b/server/internal/pacsight/repology/exporter.go new file mode 100644 index 00000000..127f9ba5 --- /dev/null +++ b/server/internal/pacsight/repology/exporter.go @@ -0,0 +1,5 @@ +package repology + +import "pacstall.dev/webserver/internal/pacsight/repology/types" + +var ExportRepologyDatabase = types.ExportRepologyDatabase diff --git a/server/internal/pacsight/repology/fetch.go b/server/internal/pacsight/repology/fetch.go new file mode 100644 index 00000000..27898493 --- /dev/null +++ b/server/internal/pacsight/repology/fetch.go @@ -0,0 +1,97 @@ +package repology + +import ( + "strings" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/array" + "pacstall.dev/webserver/pkg/common/types" +) + +func parseRepologyFilter(filter string) (string, string) { + idx := strings.Index(filter, ":") + return strings.TrimSpace(filter[:idx]), strings.TrimSpace(filter[idx+1:]) +} + +const ( + RF_REPO = "repo" + RF_SUBREPO = "subrepo" + RF_STATUS = "status" + RF_SRCNAME = "srcname" + RF_BINNAME = "binname" + RF_VERSION = "version" + RF_ORIGVERSION = "origversion" + RF_VISIBLENAME = "visiblename" + RF_SUMMARY = "summary" +) + +func FindRepologyProject(projectsMap types.RepologyApiProjectSearchResponse, search []string) (types.RepologyApiProject, error) { + if len(search) == 0 { + return types.RepologyApiProject{}, errorx.IllegalArgument.New("no search terms provided") + } + + _, projectName := parseRepologyFilter(search[0]) + + projects, ok := projectsMap[projectName] + if !ok { + return types.RepologyApiProject{}, errorx.DataUnavailable. + New("project not found"). + WithProperty(errorx.RegisterProperty("project"), projectName) + } + + for _, filter := range search[1:] { + field, value := parseRepologyFilter(filter) + + projects = array.Filter(projects, func(i *array.Iterator[types.RepologyApiProject]) bool { + switch field { + case RF_REPO: + return i.Value.Repository == value + case RF_SUBREPO: + return i.Value.SubRepository != nil && *i.Value.SubRepository == value + case RF_STATUS: + return i.Value.Status == value + case RF_SRCNAME: + return i.Value.SourceName != nil && *i.Value.SourceName == value + case RF_BINNAME: + return i.Value.BinaryName != nil && *i.Value.BinaryName == value + case RF_VERSION: + return i.Value.Version == value + case RF_ORIGVERSION: + return i.Value.OriginalVersion == value + case RF_VISIBLENAME: + return i.Value.VisibleName != nil && *i.Value.VisibleName == value + case RF_SUMMARY: + return i.Value.Summary == value + default: + return false + } + }) + } + + projects = sortByStatus(projects) + + if len(projects) == 0 { + return types.RepologyApiProject{}, errorx.IllegalArgument.New("no projects found") + } + + return projects[0], nil +} + +var repologyStatusPriority = map[string]int{ + "newest": 0, + "rolling": 1, + "devel": 3, + "legacy": 4, + "outdated": 5, + "unique": 6, + "noscheme": 7, + "incorrect": 7, + "untrusted": 7, + "ignored": 7, +} + +func sortByStatus(projects []types.RepologyApiProject) []types.RepologyApiProject { + return array.SortBy(array.Clone(projects), func(p1, p2 types.RepologyApiProject) bool { + return repologyStatusPriority[p1.Status] < repologyStatusPriority[p2.Status] + }) +} diff --git a/server/internal/pacsight/repology/scheduler.go b/server/internal/pacsight/repology/scheduler.go new file mode 100644 index 00000000..f13e7f3e --- /dev/null +++ b/server/internal/pacsight/repology/scheduler.go @@ -0,0 +1,67 @@ +package repology + +import ( + "encoding/json" + "os" + "time" + + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/types" +) + +var Packages types.RepologyApiProjectSearchResponse = make(map[string][]types.RepologyApiProject) + +func ScheduleRefresh(every time.Duration) { + log.Info("attempting to read repology cache...") + cache, err := readCache() + if err != nil { + log.Warn("no cache found. this is normal on clean runs: %+v", err) + } else { + log.Info("repology cache read successfully") + Packages = cache + } + + go func() { + for { + log.Info("refreshing Repology database...") + results, err := ExportRepologyDatabase() + if err != nil { + log.Error("failed to export Repology projects: %+v", err) + } else { + log.Info("repology database refreshed successfully") + Packages = results + if err = cacheValues(Packages); err != nil { + log.Error("failed to cache Repology projects: %+v", err) + } + } + time.Sleep(every) + } + }() + + log.Info("scheduled repology refresh every %v", every) +} + +const CACHE_FILE_NAME = "repology_cache.json" + +func readCache() (types.RepologyApiProjectSearchResponse, error) { + bytes, err := os.ReadFile(CACHE_FILE_NAME) + if err != nil { + return nil, err + } + + var cache types.RepologyApiProjectSearchResponse = make(map[string][]types.RepologyApiProject) + if err = json.Unmarshal(bytes, &cache); err != nil { + return nil, err + } + + return cache, nil +} + +func cacheValues(value types.RepologyApiProjectSearchResponse) error { + bytes, err := json.Marshal(value) + if err != nil { + return err + } + + return os.WriteFile(CACHE_FILE_NAME, bytes, 0644) +} diff --git a/server/repology/internal/api.go b/server/internal/pacsight/repology/types/api.go similarity index 83% rename from server/repology/internal/api.go rename to server/internal/pacsight/repology/types/api.go index 6201a441..6ba33d0b 100644 --- a/server/repology/internal/api.go +++ b/server/internal/pacsight/repology/types/api.go @@ -1,4 +1,4 @@ -package internal +package types import ( "encoding/json" @@ -7,12 +7,13 @@ import ( "net/url" "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/types" ) const _USER_AGENT = "Pacstall/WebServer/Exporter" -func getProjectSearch(projectName string) (RepologyApiProjectSearchResponse, error) { - var response RepologyApiProjectSearchResponse +func getProjectSearch(projectName string) (types.RepologyApiProjectSearchResponse, error) { + var response types.RepologyApiProjectSearchResponse request := http.Request{ Method: "GET", diff --git a/server/internal/pacsight/repology/types/exporter.go b/server/internal/pacsight/repology/types/exporter.go new file mode 100644 index 00000000..2f34933b --- /dev/null +++ b/server/internal/pacsight/repology/types/exporter.go @@ -0,0 +1,98 @@ +package types + +import ( + "math" + "strings" + "time" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/pkg/common/log" + "pacstall.dev/webserver/pkg/common/types" +) + +// Ends up waiting `sum(1^2, 2^2, 3^2, ..., RETRY_COUNT^2)` = 385 seconds (6.4 minutes) at most +const RETRY_COUNT = 10 +const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second + +func ExportRepologyDatabase() (types.RepologyApiProjectSearchResponse, error) { + page := 1 + lastProjectName := "" + + lastRepoFetch := time.Now() + var repologyProjectsMap = make(types.RepologyApiProjectSearchResponse) + + for { + if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { + time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) + } + + log.Debug("page %v | cursor at: %v", page, lastProjectName) + + var projectPage map[string][]types.RepologyApiProject + var err error + + retry: + for i := 1; i <= RETRY_COUNT; i += 1 { + projectPage, err = getProjectSearch(lastProjectName) + if err == nil { + break retry + } + + retryDelay := time.Duration(math.Pow(float64(i), 2)) * REPOLOGY_PROJECT_FETCH_THROTTLE + log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) + time.Sleep(retryDelay) + } + + if err != nil { + return nil, errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) + } + + lastRepoFetch = time.Now() + + shouldStop := false + for projectName, apiProjectProvider := range projectPage { + if _, ok := repologyProjectsMap[projectName]; !ok { + repologyProjectsMap[projectName] = []types.RepologyApiProject{} + } + + repologyProjectsMap[projectName] = append(repologyProjectsMap[projectName], apiProjectProvider...) + + shouldStop = projectName == lastProjectName + lastProjectName = identityOrSkipProject(projectName) + } + + if shouldStop { + break + } + + page += 1 + } + + return repologyProjectsMap, nil +} + +var projectNamesToSkipToNextCussor = map[string]string{ + "emacs:": "emacsa", + "go:": "goa", + "haskell:": "haskella", + "lisp:": "lispa", + "node:": "nodea", + "ocaml:": "ocamla", + "perl:": "perla", + "php:": "phpa", + "python:": "pythona", + "r:": "ra", + "ruby:": "rubya", + "rust:": "rusta", + "texlive:": "texlivea", +} + +func identityOrSkipProject(name string) string { + for prefix, skipTo := range projectNamesToSkipToNextCussor { + if strings.HasPrefix(name, prefix) { + return skipTo + } + } + + return name +} diff --git a/server/internal/pacsight/rpcall/service.go b/server/internal/pacsight/rpcall/service.go new file mode 100644 index 00000000..d46fddf1 --- /dev/null +++ b/server/internal/pacsight/rpcall/service.go @@ -0,0 +1,26 @@ +package rpcall + +import ( + "net/rpc" + + "github.com/joomcode/errorx" + "pacstall.dev/webserver/internal/pacsight/repology" + "pacstall.dev/webserver/pkg/common/pacsight" +) + +func RegisterService() { + rpc.Register(new(Service)) + rpc.HandleHTTP() +} + +type Service int + +func (s *Service) GetRepologyProject(args *pacsight.GetRepologyProjectArgs, reply *pacsight.GetRepologyProjectReply) error { + project, err := repology.FindRepologyProject(repology.Packages, args.Filters) + if err != nil { + return errorx.ExternalError.Wrap(err, "pacsight rpc call failed") + } + + reply.Project = project + return nil +} diff --git a/server/types/array/array.go b/server/pkg/common/array/array.go similarity index 100% rename from server/types/array/array.go rename to server/pkg/common/array/array.go diff --git a/server/types/array/array_test.go b/server/pkg/common/array/array_test.go similarity index 100% rename from server/types/array/array_test.go rename to server/pkg/common/array/array_test.go diff --git a/server/types/array/sort.go b/server/pkg/common/array/sort.go similarity index 100% rename from server/types/array/sort.go rename to server/pkg/common/array/sort.go diff --git a/server/pkg/common/config/build/vars.go b/server/pkg/common/config/build/vars.go new file mode 100644 index 00000000..5a88ce0f --- /dev/null +++ b/server/pkg/common/config/build/vars.go @@ -0,0 +1,4 @@ +package build + +var Production = "false" +var Version = "development" diff --git a/server/pkg/common/config/vars.go b/server/pkg/common/config/vars.go new file mode 100644 index 00000000..d20d83ba --- /dev/null +++ b/server/pkg/common/config/vars.go @@ -0,0 +1,8 @@ +package config + +import ( + "pacstall.dev/webserver/pkg/common/config/build" +) + +var Production = build.Production == "true" || build.Production == "1" +var Version = build.Version diff --git a/server/pkg/common/env/env.go b/server/pkg/common/env/env.go new file mode 100644 index 00000000..6237c973 --- /dev/null +++ b/server/pkg/common/env/env.go @@ -0,0 +1,119 @@ +package env + +import ( + "time" + + "pacstall.dev/webserver/pkg/common/config/build" +) + +var PacSight = struct { + Port int +}{} + +func initPacSightEnv() { + PacSight.Port = GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) +} + +var PacNexus = struct { + Port int + PublicDir string + Version string + Production bool +}{} + +func initPacNexusEnv() { + PacNexus.Port = GetEnvIntOrDefault("PACSTALL_PACNEXUS_PORT", 3300) + PacNexus.PublicDir = GetEnvStringOrDefault("PACSTALL_PACNEXUS_PUBLIC_DIR", "./public") + PacNexus.Version = build.Version + PacNexus.Production = toBool(build.Production) +} + +// Configuration for the discord integration +var Discord = struct { + Token string + ChannelID string + Enabled bool + Tags string +}{} + +func initDiscordEnv() { + Discord.Enabled = GetEnvBoolOrDefault("PACSTALL_DISCORD_ENABLED", false) + + if !Discord.Enabled { + return + } + + Discord.Token = GetEnvString("PACSTALL_DISCORD_TOKEN") + Discord.ChannelID = GetEnvString("PACSTALL_DISCORD_CHANNEL_ID") + Discord.Tags = GetEnvString("PACSTALL_DISCORD_TAGS") +} + +var PacstallPrograms = struct { + Branch string + UpdateInterval time.Duration + TempDir string + MaxOpenFiles int + GitURL string + GitClonePath string +}{} + +func initPacstallProgramsEnv() { + PacstallPrograms.Branch = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_BRANCH", "master") + PacstallPrograms.UpdateInterval = time.Duration(GetEnvIntOrDefault("PACSTALL_PROGRAMS_UPDATE_INTERVAL", 15*60)) * time.Second + PacstallPrograms.TempDir = GetEnvStringOrDefault("PACSTALL_PROGRAMS_TEMP_DIR", "./tmp") + PacstallPrograms.MaxOpenFiles = GetEnvIntOrDefault("PACSTALL_PROGRAMS_MAX_OPEN_FILES", 100) + PacstallPrograms.GitURL = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_URL", "https://github.com/pacstall/pacstall-programs.git") + PacstallPrograms.GitClonePath = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_CLONE_PATH", "./programs") +} + +// Configuration for the database +var Database = struct { + Host string + Port int + User string + Password string + Name string +}{} + +func initDatabaseEnv() { + Database.Host = GetEnvStringOrDefault("PACSTALL_DATABASE_HOST", "localhost") + Database.Port = GetEnvIntOrDefault("PACSTALL_DATABASE_PORT", 3306) + Database.User = GetEnvStringOrDefault("PACSTALL_DATABASE_USER", "root") + Database.Password = GetEnvStringOrDefault("PACSTALL_DATABASE_PASSWORD", "changeme") + Database.Name = GetEnvStringOrDefault("PACSTALL_DATABASE_NAME", "pacstall") +} + +// Configuration for the Matomo API +var Matomo = struct { + Enabled bool +}{} + +func initMatomoEnv() { + Matomo.Enabled = GetEnvBoolOrDefault("PACSTALL_MATOMO_ENABLED", false) +} + +// Configuration for the Repology API +var Repology = struct { + Enabled bool + RepologyUpdateInterval time.Duration +}{} + +func initRepologyEnv() { + Repology.Enabled = GetEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", false) + + if !Repology.Enabled { + return + } + + Repology.RepologyUpdateInterval = time.Duration(GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second +} + +func Init() { + initPacNexusEnv() + initPacSightEnv() + initDatabaseEnv() + initDiscordEnv() + initMatomoEnv() + initPacstallProgramsEnv() + initRepologyEnv() +} diff --git a/server/config/util.go b/server/pkg/common/env/util.go similarity index 60% rename from server/config/util.go rename to server/pkg/common/env/util.go index 5d5ec124..051bca49 100644 --- a/server/config/util.go +++ b/server/pkg/common/env/util.go @@ -1,9 +1,10 @@ -package config +package env import ( "fmt" "os" "strconv" + "strings" ) func toInt(str string) int { @@ -59,34 +60,59 @@ func getEnvVar[T any](key string, format formatter[T]) T { panic(fmt.Sprintf("could not find environment variable '%s'", key)) } - return format.Format(val) + formattedValue := format.Format(val) + printEnvVariable(key, formattedValue) + + return formattedValue +} + +func printEnvVariable(key string, value interface{}) { + key = strings.ToUpper(key) + if strings.Contains(key, "PASSWORD") || strings.Contains(key, "PRIVATE") || strings.Contains(key, "SECRET") || strings.Contains(key, "HIDDEN") { + fmt.Printf("env[%s]=%s\n", key, "********") + return + } + + fmt.Printf("env[%s]=%#v\n", key, value) } func getEnvOrDefault[T any](key string, defaultValue T, format formatter[T]) T { val, ok := os.LookupEnv(key) if !ok { + printEnvVariable(key, defaultValue) return defaultValue } - return format.Format(val) + formattedValue := format.Format(val) + printEnvVariable(key, formattedValue) + + return formattedValue } var _stringFormatter formatter[string] = stringFormatter{} var _intFormatter formatter[int] = intFormatter{} var _boolFormatter formatter[bool] = boolFormatter{} -func getEnvString(key string) string { +func GetEnvString(key string) string { return getEnvVar(key, _stringFormatter) } -func getEnvInt(key string) int { +func GetEnvStringOrDefault(key, defaultValue string) string { + return getEnvOrDefault(key, defaultValue, _stringFormatter) +} + +func GetEnvInt(key string) int { return getEnvVar(key, _intFormatter) } -func getEnvBool(key string) bool { +func GetEnvIntOrDefault(key string, defaultValue int) int { + return getEnvOrDefault(key, defaultValue, _intFormatter) +} + +func GetEnvBool(key string) bool { return getEnvVar(key, _boolFormatter) } -func getEnvBoolOrDefault(key string, defaultValue bool) bool { +func GetEnvBoolOrDefault(key string, defaultValue bool) bool { return getEnvOrDefault(key, defaultValue, _boolFormatter) } diff --git a/server/log/discord.go b/server/pkg/common/log/discord.go similarity index 100% rename from server/log/discord.go rename to server/pkg/common/log/discord.go diff --git a/server/log/lib.go b/server/pkg/common/log/lib.go similarity index 53% rename from server/log/lib.go rename to server/pkg/common/log/lib.go index 189325be..b6bc6e89 100644 --- a/server/log/lib.go +++ b/server/pkg/common/log/lib.go @@ -5,9 +5,7 @@ import ( glog "log" "os" - "github.com/bwmarrin/discordgo" "github.com/fatih/color" - "pacstall.dev/webserver/config" ) var ( @@ -18,13 +16,6 @@ var ( logWarn = color.YellowString("WARN") ) -const ( - logDiscordError = "❌ Error ❌" - logDiscordWarn = "⚠️ Warning" - logDiscordFatal = "💀☢️💥 Fatal 🪦⚰️🧟‍♂️" - logDiscordNotify = "📢 Notification" -) - type tLogLevel uint8 var Level = struct { @@ -73,7 +64,6 @@ func Error(message string, args ...any) { } doLog(logError, message, args...) - go sendDiscordMessage(true, logDiscordError, message, args...) } func Fatal(message string, args ...any) { @@ -81,7 +71,6 @@ func Fatal(message string, args ...any) { return } - sendDiscordMessage(true, logDiscordFatal, message, args...) doLog(logFatal, message, args...) } @@ -91,7 +80,6 @@ func Warn(message string, args ...any) { } doLog(logWarn, message, args...) - go sendDiscordMessage(true, logDiscordWarn, message, args...) } func Debug(message string, args ...any) { @@ -101,39 +89,3 @@ func Debug(message string, args ...any) { doLog(logDebug, message, args...) } - -func Notify(message string, args ...any) { - go sendDiscordMessage(false, logDiscordNotify, message, args...) -} - -func NotifyCustom(level, message string, args ...any) { - go sendDiscordMessage(false, level, message, args...) -} - -func sendDiscordMessage(tag bool, level, message string, args ...any) { - if !config.Discord.Enabled { - return - } - - msg := fmt.Sprintf("Webserver - %s: %s\n", level, fmt.Sprintf(message, args...)) - if tag { - msg = fmt.Sprintf("%s %s", config.Discord.Tags, msg) - } - - _, err := discordClient.ChannelMessageSend( - config.Discord.ChannelID, - msg, - ) - - if err != nil { - panic(fmt.Sprintf("failed to send discord message\n%v", err)) - } -} - -var discordClient = func() *discordgo.Session { - if config.Discord.Enabled { - return connect(config.Discord.Token) - } - - return nil -}() diff --git a/server/pkg/common/pacsight/rpc.go b/server/pkg/common/pacsight/rpc.go new file mode 100644 index 00000000..37a820d9 --- /dev/null +++ b/server/pkg/common/pacsight/rpc.go @@ -0,0 +1,42 @@ +package pacsight + +import ( + "net/rpc" + "strconv" + + "pacstall.dev/webserver/internal/pacnexus/config" + "pacstall.dev/webserver/pkg/common/types" +) + +type GetRepologyProjectArgs struct { + Filters []string +} + +type GetRepologyProjectReply struct { + Project types.RepologyApiProject +} + +type PacsightRpcService struct { + client *rpc.Client +} + +func NewPacsightRpcService(addr string, port int) (*PacsightRpcService, error) { + if !config.Repology.Enabled { + return nil, nil + } + + client, err := rpc.DialHTTP("tcp", addr+":"+strconv.FormatInt(int64(port), 10)) + return &PacsightRpcService{client}, err +} + +func (s *PacsightRpcService) GetRepologyProject(filters []string) (types.RepologyApiProject, error) { + args := &GetRepologyProjectArgs{Filters: filters} + reply := &GetRepologyProjectReply{} + + err := s.client.Call("Service.GetRepologyProject", args, reply) + if err != nil { + return types.RepologyApiProject{}, err + } + + return reply.Project, nil +} diff --git a/server/types/equals.go b/server/pkg/common/types/equals.go similarity index 100% rename from server/types/equals.go rename to server/pkg/common/types/equals.go diff --git a/server/types/percentage.go b/server/pkg/common/types/percentage.go similarity index 100% rename from server/types/percentage.go rename to server/pkg/common/types/percentage.go diff --git a/server/types/pkgtype.go b/server/pkg/common/types/pkgtype.go similarity index 100% rename from server/types/pkgtype.go rename to server/pkg/common/types/pkgtype.go diff --git a/server/repology/internal/api_types.go b/server/pkg/common/types/repology.go similarity index 97% rename from server/repology/internal/api_types.go rename to server/pkg/common/types/repology.go index 57d48214..89bd5a81 100644 --- a/server/repology/internal/api_types.go +++ b/server/pkg/common/types/repology.go @@ -1,4 +1,4 @@ -package internal +package types type RepologyApiProject struct { Repository string `json:"repo"` diff --git a/server/repology/exporter.go b/server/repology/exporter.go deleted file mode 100644 index 177add9f..00000000 --- a/server/repology/exporter.go +++ /dev/null @@ -1,5 +0,0 @@ -package repology - -import "pacstall.dev/webserver/repology/internal" - -var ExportRepologyDatabase = internal.ExportRepologyDatabase diff --git a/server/repology/fetch.go b/server/repology/fetch.go deleted file mode 100644 index 71b29024..00000000 --- a/server/repology/fetch.go +++ /dev/null @@ -1,78 +0,0 @@ -package repology - -import ( - "fmt" - "strings" - - "github.com/joomcode/errorx" - "pacstall.dev/webserver/model" - "pacstall.dev/webserver/types/array" -) - -func parseRepologyFilter(filter string) (string, string) { - idx := strings.Index(filter, ":") - return strings.TrimSpace(filter[:idx]), strings.TrimSpace(filter[idx+1:]) -} - -var repologyFilterToColumn = map[string]string{ - "repo": model.RepologyProjectProviderColumns.Repository, - "subrepo": model.RepologyProjectProviderColumns.SubRepository, - "status": model.RepologyProjectProviderColumns.Status, - "srcname": model.RepologyProjectProviderColumns.SourceName, - "binname": model.RepologyProjectProviderColumns.BinaryName, - "version": model.RepologyProjectProviderColumns.Version, - "origversion": model.RepologyProjectProviderColumns.OriginalVersion, - "visiblename": model.RepologyProjectProviderColumns.VisibleName, - "summary": model.RepologyProjectProviderColumns.Summary, -} - -func findRepologyProject(search []string) (model.RepologyProjectProvider, error) { - var result model.RepologyProjectProvider - - if len(search) == 0 { - return result, errorx.IllegalArgument.New("no search terms provided") - } - - db := model.Instance() - _, projectName := parseRepologyFilter(search[0]) - - query := db.Where("project_name = ?", projectName).Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true) - for _, filter := range search[1:] { - filterName, filterValue := parseRepologyFilter(filter) - column, ok := repologyFilterToColumn[filterName] - if !ok { - return result, errorx.IllegalArgument.New("invalid filter '%v'", filterName) - } - - query = query.Where(fmt.Sprintf("%v = ?", column), filterValue) - } - - var results []model.RepologyProjectProvider - if err := query.Order("version desc").Find(&results).Error; err != nil || len(results) == 0 { - return result, errorx.Decorate(err, "failed to fetch repology project") - } - - results = sortByStatus(results) - result = results[0] - - return result, nil -} - -var repologyStatusPriority = map[string]int{ - "newest": 0, - "rolling": 1, - "devel": 3, - "legacy": 4, - "outdated": 5, - "unique": 6, - "noscheme": 7, - "incorrect": 7, - "untrusted": 7, - "ignored": 7, -} - -func sortByStatus(projects []model.RepologyProjectProvider) []model.RepologyProjectProvider { - return array.SortBy(array.Clone(projects), func(p1, p2 model.RepologyProjectProvider) bool { - return repologyStatusPriority[p1.Status] < repologyStatusPriority[p2.Status] - }) -} diff --git a/server/repology/internal/exporter.go b/server/repology/internal/exporter.go deleted file mode 100644 index a55ecd02..00000000 --- a/server/repology/internal/exporter.go +++ /dev/null @@ -1,186 +0,0 @@ -package internal - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/joomcode/errorx" - "gorm.io/gorm" - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" -) - -const RETRY_COUNT = 5 - -func ExportRepologyDatabase(db *gorm.DB) error { - err := migrateTables(db) - if err != nil { - return errors.Join(errors.New("failed to reset repology tables"), err) - } - - it := 1 - lastProjectName := "" - - const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second - - lastRepoFetch := time.Now() - - for { - if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { - time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) - } - - log.Debug("page %v | cursor at: %v", it, lastProjectName) - - var projectPage map[string][]RepologyApiProject - var err error - - retry: - for i := 0; i < RETRY_COUNT; i += 1 { - projectPage, err = getProjectSearch(lastProjectName) - if err == nil { - break retry - } - - retryDelay := time.Duration(i+1) * REPOLOGY_PROJECT_FETCH_THROTTLE - log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) - time.Sleep(retryDelay) - } - - if err != nil { - return errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) - } - - lastRepoFetch = time.Now() - - var projects []model.RepologyProject - var projectProviders []model.RepologyProjectProvider - for projectName, apiProjectProvider := range projectPage { - - lastProjectName = identityOrSkipProject(projectName) - for _, apiProjectProvider := range apiProjectProvider { - // Save project provider as inactive - projectProvider := mapRepologyApiProjectProviderToModel(projectName, apiProjectProvider) - projectProviders = append(projectProviders, projectProvider) - - project := model.RepologyProject{ - Name: projectName, - } - - projects = append(projects, project) - } - } - - if len(projects) <= 1 { - break - } - - err = db.Save(&projects).Error - if err != nil { - return errors.Join(errors.New("failed to create repology projects"), err) - } - - err = db.CreateInBatches(&projectProviders, 90).Error - if err != nil { - return errors.Join(errors.New("failed to create repology project providers"), err) - } - - it++ - } - - // Delete active (old) repology project providers - if err := db.Debug().Where(fmt.Sprintf("%v = ?", model.RepologyProjectProviderColumns.Active), true).Delete(&model.RepologyProjectProvider{}).Error; err != nil { - return errors.Join(errors.New("failed to delete old repology project providers"), err) - } - - // Mark new repology project providers as active - if err := db.Debug().Exec( - fmt.Sprintf( - "UPDATE %s SET %s = 1", - model.RepologyProjectProviderTableName, - model.RepologyProjectProviderColumns.Active, - ), - ).Error; err != nil { - return errors.Join(errors.New("failed to update new repology project providers"), err) - } - - return nil -} - -var projectNamesToSkipToNextCussor = map[string]string{ - "emacs:": "emacsa", - "go:": "goa", - "haskell:": "haskella", - "lisp:": "lispa", - "node:": "nodea", - "ocaml:": "ocamla", - "perl:": "perla", - "php:": "phpa", - "python:": "pythona", - "r:": "ra", - "ruby:": "rubya", - "rust:": "rusta", - "texlive:": "texlivea", -} - -func identityOrSkipProject(name string) string { - for prefix, skipTo := range projectNamesToSkipToNextCussor { - if strings.HasPrefix(name, prefix) { - return skipTo - } - } - - return name -} - -func migrateTables(db *gorm.DB) error { - err := db.AutoMigrate(&model.RepologyProject{}) - if err != nil { - return err - } - - if err = truncateTable(db, model.RepologyProjectTableName); err != nil { - return err - } - - err = db.AutoMigrate(&model.RepologyProjectProvider{}) - if err != nil { - return err - } - - if err = truncateTable(db, model.RepologyProjectProviderTableName); err != nil { - return err - } - - return nil -} - -func truncateTable(db *gorm.DB, tableName string) error { - log.Debug("attempting to truncate table %v", tableName) - err := db.Exec("TRUNCATE TABLE " + tableName).Error - if err != nil { - return errors.Join(fmt.Errorf("failed to truncate table %v", tableName), err) - } - - log.Info("successfully truncated table %v", tableName) - return nil -} - -func mapRepologyApiProjectProviderToModel(projectName string, apiProjectProvider RepologyApiProject) model.RepologyProjectProvider { - projectProvider := model.RepologyProjectProvider{ - ProjectName: projectName, - Repository: apiProjectProvider.Repository, - SubRepository: apiProjectProvider.SubRepository, - SourceName: apiProjectProvider.SourceName, - VisibleName: apiProjectProvider.VisibleName, - BinaryName: apiProjectProvider.BinaryName, - Version: apiProjectProvider.Version, - OriginalVersion: apiProjectProvider.OriginalVersion, - Status: apiProjectProvider.Status, - Summary: apiProjectProvider.Summary, - Active: false, - } - return projectProvider -} diff --git a/server/repology/lib.go b/server/repology/lib.go deleted file mode 100644 index 9153cdc1..00000000 --- a/server/repology/lib.go +++ /dev/null @@ -1,18 +0,0 @@ -package repology - -import ( - "pacstall.dev/webserver/types/pac" -) - -func Sync(script *pac.Script) error { - if len(script.Repology) == 0 { - return nil - } - - project, err := findRepologyProject(script.Repology) - if err != nil { - return err - } - - return updateScriptVersion(project, script) -} diff --git a/server/repology/scheduler.go b/server/repology/scheduler.go deleted file mode 100644 index b917befb..00000000 --- a/server/repology/scheduler.go +++ /dev/null @@ -1,25 +0,0 @@ -package repology - -import ( - "time" - - "pacstall.dev/webserver/log" - "pacstall.dev/webserver/model" -) - -func ScheduleRefresh(every time.Duration) { - db := model.Instance() - go func() { - - for { - log.Info("refreshing Repology database...") - err := ExportRepologyDatabase(db) - if err != nil { - log.Error("failed to export Repology projects: %+v", err) - } else { - log.Info("repology database refreshed successfully") - } - time.Sleep(every) - } - }() -} diff --git a/server/types/pac/parser/parse_test.go b/server/types/pac/parser/parse_test.go deleted file mode 100644 index 984664ce..00000000 --- a/server/types/pac/parser/parse_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package parser_test - -import ( - "encoding/json" - "fmt" - "os" - "path" - "testing" - - "pacstall.dev/webserver/types" - "pacstall.dev/webserver/types/pac" - "pacstall.dev/webserver/types/pac/parser" - "pacstall.dev/webserver/types/pac/parser/pacsh" -) - -var FIXTURES_DIR = func() string { - dir, err := os.Getwd() - if err != nil { - panic(err) - } - - return path.Join(dir, "../../../fixtures") -}() - -var TEST_PROGRAMS_DIR = path.Join(FIXTURES_DIR, "test-programs") - -func assertEquals(t *testing.T, what string, expected interface{}, actual interface{}) { - if actual != expected { - t.Errorf("pacscript.%v: expected '%#v', got '%#v'", what, expected, actual) - } -} - -func assertArrayEquals[T types.Equaller](t *testing.T, what string, expected []T, actual []T) { - if len(actual) == len(expected) && len(actual) == 0 { - return - } - - if len(actual) != len(expected) { - t.Errorf("pacscript.%v expected len '%v', got len '%v' (expected '%#v', got '%#v')", what, len(expected), len(actual), expected, actual) - return - } - - for idx := range expected { - if !expected[idx].Equals(actual[idx]) { - t.Errorf("pacscript.%v[%v] expected '%#v', got '%#v'", what, idx, expected, actual) - } - } -} - -func assertStringArrayEquals(t *testing.T, what string, expected []string, actual []string) { - if len(actual) == len(expected) && len(actual) == 0 { - return - } - - if len(actual) != len(expected) { - t.Errorf("pacscript.%v expected len '%v', got len '%v' (expected '%#v', got '%#v')", what, len(expected), len(actual), expected, actual) - return - } - - for idx := range expected { - if expected[idx] != actual[idx] { - t.Errorf("pacscript.%v[%v] expected '%#v', got '%#v'", what, idx, expected, actual) - } - } -} - -func assertPacscriptEquals(t *testing.T, expected *pac.Script, actual *pac.Script) { - assertEquals(t, "package name", expected.PackageName, actual.PackageName) - assertStringArrayEquals(t, "maintainers", expected.Maintainers, actual.Maintainers) - assertEquals(t, "description", expected.Description, actual.Description) - assertEquals(t, "gives", expected.Gives, actual.Gives) - assertEquals(t, "version", expected.Version, actual.Version) - assertArrayEquals(t, "breaks", expected.Breaks, actual.Breaks) - assertArrayEquals(t, "conflicts", expected.Conflicts, actual.Conflicts) - assertArrayEquals(t, "replaces", expected.Replaces, actual.Replaces) - assertEquals(t, "pretty name", expected.PrettyName, actual.PrettyName) - assertArrayEquals(t, "sources", expected.Source, actual.Source) - assertArrayEquals(t, "runtime dependencies", expected.RuntimeDependencies, actual.RuntimeDependencies) - assertArrayEquals(t, "build dependencies", expected.BuildDependencies, actual.BuildDependencies) - assertArrayEquals(t, "optional dependencies", expected.OptionalDependencies, actual.OptionalDependencies) - assertArrayEquals(t, "pacstall dependencies", expected.PacstallDependencies, actual.PacstallDependencies) - assertStringArrayEquals(t, "required by", expected.RequiredBy, actual.RequiredBy) - assertStringArrayEquals(t, "repology", expected.Repology, actual.Repology) - assertEquals(t, "update status", expected.UpdateStatus, actual.UpdateStatus) -} - -func loadSnapshot(snapshotPath string) (*pac.Script, error) { - bytes, err := os.ReadFile(snapshotPath) - if err != nil { - return nil, err - } - - var out pac.Script - if err := json.Unmarshal(bytes, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func assertPacscriptMatchesSnapshot(t *testing.T, pkgname string) { - t.Helper() - - if pacsh.CreateTempDirectory("./tmp") != nil { - t.Errorf("failed to create temp directory") - return - } - - actual, err := parser.ParsePacscriptFile(TEST_PROGRAMS_DIR, pkgname) - if err != nil { - t.Errorf("expected no error, got %v", err) - return - } - - snapshotPath := path.Join(TEST_PROGRAMS_DIR, "packages", pkgname, fmt.Sprintf("%v.snapshot.json", pkgname)) - expected, err := loadSnapshot(snapshotPath) - if err != nil { - bytes, err := json.Marshal(actual) - if err != nil { - t.Errorf("failed to serialize snapshot. %v", err) - return - } - - if err := os.WriteFile(snapshotPath, bytes, 0644); err != nil { - t.Errorf("failed to write snapshot. %v", err) - return - } - - t.Errorf("missing snapshot. a new one has been generated. rerun tests") - return - } - - assertPacscriptEquals(t, expected, actual) -} - -func Test_PacscriptSnapshots(t *testing.T) { - dirEntries, err := os.ReadDir(path.Join(TEST_PROGRAMS_DIR, "packages")) - if err != nil { - t.Errorf("failed to read test packages. %v", err) - return - } - - for _, dirEntry := range dirEntries { - if !dirEntry.IsDir() { - continue - } - - t.Logf("==> Running snapshot test for file: %v", dirEntry.Name()) - assertPacscriptMatchesSnapshot(t, dirEntry.Name()) - } -} diff --git a/server/types/pac/parser/scheduler.go b/server/types/pac/parser/scheduler.go deleted file mode 100644 index 9edd7925..00000000 --- a/server/types/pac/parser/scheduler.go +++ /dev/null @@ -1,27 +0,0 @@ -package parser - -import ( - "time" - - "pacstall.dev/webserver/log" -) - -func ScheduleRefresh(every time.Duration) { - go refresh(every) -} - -func refresh(every time.Duration) { - for { - err := ParseAll() - if err != nil { - log.Error("parse error: %+v", err) - - retryIn := time.Second * 30 - log.Info("retrying in %v", retryIn) - time.Sleep(retryIn) - continue - } - - time.Sleep(every) - } -} From f881c79bb573a717eab98b0407abbf6b3abf405b Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Mon, 23 Sep 2024 15:49:47 +0300 Subject: [PATCH 13/17] chore: remove discord logs --- server/pkg/common/log/discord.go | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 server/pkg/common/log/discord.go diff --git a/server/pkg/common/log/discord.go b/server/pkg/common/log/discord.go deleted file mode 100644 index 02b1f33c..00000000 --- a/server/pkg/common/log/discord.go +++ /dev/null @@ -1,16 +0,0 @@ -package log - -import ( - "log" - - "github.com/bwmarrin/discordgo" -) - -func connect(token string) *discordgo.Session { - client, err := discordgo.New("Bot " + token) - if err != nil { - log.Fatalf("failed to connect to discord\n%v\n", err) - } - - return client -} From f35d8cc70bfcc510d06bfef81dc0de6f19d54c51 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Mon, 23 Sep 2024 20:34:44 +0300 Subject: [PATCH 14/17] chore: update docker compose --- docker-compose.dev.yml | 16 ++- docker-compose.yml | 31 +++-- Dockerfile => pacnexus.Dockerfile | 2 +- pacsight.Dockerfile | 31 +++++ server/Makefile | 2 +- server/cmd/pacnexus/main.go | 12 +- server/internal/pacnexus/config/env.go | 4 +- server/internal/pacsight/config/env.go | 4 +- .../internal/pacsight/repology/scheduler.go | 21 +++- .../pacsight/repology/types/exporter.go | 2 +- server/pkg/common/config/vars.go | 3 + server/pkg/common/env/env.go | 119 ------------------ server/pkg/common/log/lib.go | 59 ++++++--- 13 files changed, 140 insertions(+), 166 deletions(-) rename Dockerfile => pacnexus.Dockerfile (91%) create mode 100644 pacsight.Dockerfile delete mode 100644 server/pkg/common/env/env.go diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5792e289..7325a554 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -5,13 +5,25 @@ services: mariadb: image: mariadb # inherited from docker-compose.yml - webserver: + pacnexus: # overrides the image used from docker-compose.yml build: context: . - dockerfile: Dockerfile + dockerfile: pacnexus.Dockerfile args: VITE_VERSION: "${VITE_VERSION}" + environment: + PACSTALL_LOG_LEVEL: "debug" + # inherited from docker-compose.yml + pacsight: + # overrides the image used from docker-compose.yml + build: + context: . + dockerfile: pacsight.Dockerfile + args: + VITE_VERSION: "${VITE_VERSION}" + environment: + PACSTALL_LOG_LEVEL: "debug" # inherited from docker-compose.yml matomo: diff --git a/docker-compose.yml b/docker-compose.yml index 909d2f75..27f4d541 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.2' - services: mariadb: @@ -14,7 +12,7 @@ services: - target: 3306 published: 3306 - webserver: + pacnexus: image: ghcr.io/pacstall/webserver:latest depends_on: - mariadb @@ -28,10 +26,8 @@ services: max-file: "5" max-size: "10m" environment: - PACSTALL_DISCORD_TOKEN: "" - PACSTALL_DISCORD_CHANNEL_ID: "" - PACSTALL_DISCORD_ENABLED: "false" - PACSTALL_DISCORD_TAGS: "" # See https://discord.com/developers/docs/reference#message-formatting + PACSTALL_PACSIGHT_PORT: 3301 + PACSTALL_PACSIGHT_HOST: pacsight PACSTALL_DATABASE_HOST: mariadb PACSTALL_DATABASE_USER: root PACSTALL_DATABASE_PASSWORD: changeme @@ -40,7 +36,25 @@ services: PACSTALL_MATOMO_ENABLED: "true" MATOMO_DOMAIN: "http://matomo" MATOMO_SITE_ID: "1" - PACSTALL_PROGRAMS_GIT_BRANCH: "master" + PACSTALL_LOG_LEVEL: "info" + + pacsight: + image: ghcr.io/pacstall/pacsight:latest + logging: + driver: "json-file" + options: + max-file: "5" + max-size: "10m" + environment: + PACSTALL_PACSIGHT_PORT: 3301 + PACSTALL_REPOLOGY_CACHE_PATH: "/var/cache/pacsight" + PACSTALL_LOG_LEVEL: "info" + deploy: + resources: + limits: + memory: 6g + volumes: + - pacsight_cache:/var/cache/pacsight matomo: image: matomo @@ -60,3 +74,4 @@ services: volumes: db: matomo: + pacsight_cache: \ No newline at end of file diff --git a/Dockerfile b/pacnexus.Dockerfile similarity index 91% rename from Dockerfile rename to pacnexus.Dockerfile index ff1aac44..ca7019fb 100644 --- a/Dockerfile +++ b/pacnexus.Dockerfile @@ -41,5 +41,5 @@ WORKDIR /root/dist/ RUN ls -al /root/dist -CMD echo "Starting webserver in a few seconds..." && sleep 3 && "./webserver" +CMD echo "Starting pacnexus in a few seconds..." && sleep 3 && "./pacnexus" EXPOSE 3300 diff --git a/pacsight.Dockerfile b/pacsight.Dockerfile new file mode 100644 index 00000000..2e8cd080 --- /dev/null +++ b/pacsight.Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.23-alpine AS server + +ARG VITE_VERSION +ENV VITE_VERSION="${VITE_VERSION}" +ENV NODE_ENV="production" + +WORKDIR /root/ + +COPY ./server ./server + +RUN apk add --no-cache make gcc musl-dev + +WORKDIR /root/server +RUN make dist/pacsight + +FROM ubuntu:22.04 +WORKDIR /root/ + +RUN apt update +RUN apt install wget curl -y + +COPY --from=server /root/server/dist/ /root/server/dist/ + +RUN apt update && apt install make git jq -y + +WORKDIR /root/server/dist + +RUN ls -al /root/server/dist + +CMD "./pacsight" +EXPOSE 8080 diff --git a/server/Makefile b/server/Makefile index a48a9cfe..e2002640 100644 --- a/server/Makefile +++ b/server/Makefile @@ -42,4 +42,4 @@ clean: @echo "Cleaned up." fmt: - go fmt ./... + \ No newline at end of file diff --git a/server/cmd/pacnexus/main.go b/server/cmd/pacnexus/main.go index 08e87645..b6e69a1d 100644 --- a/server/cmd/pacnexus/main.go +++ b/server/cmd/pacnexus/main.go @@ -12,7 +12,6 @@ import ( urlshortener "pacstall.dev/webserver/internal/pacnexus/server/api/url_shortener" pac_ssr "pacstall.dev/webserver/internal/pacnexus/server/ssr/pacscript" "pacstall.dev/webserver/internal/pacnexus/types/pac/parser" - globalConfig "pacstall.dev/webserver/pkg/common/config" "pacstall.dev/webserver/pkg/common/log" "pacstall.dev/webserver/pkg/common/pacsight" ) @@ -49,22 +48,15 @@ func setupRequests() { } func main() { - config.Init() - - if globalConfig.Production { - log.SetLogLevel(log.Level.Info) - } else { - log.SetLogLevel(log.Level.Debug) - } - startedAt := time.Now() + config.Init() printLogo() setupRequests() log.Info("registered http requests") - pacsightRpc, err := pacsight.NewPacsightRpcService("localhost", 8080) + pacsightRpc, err := pacsight.NewPacsightRpcService(config.PacSight.Host, config.PacSight.Port) if err != nil { log.Error("failed to create pacsight rpc service: %+v", err) return diff --git a/server/internal/pacnexus/config/env.go b/server/internal/pacnexus/config/env.go index 7228c610..a183dfeb 100644 --- a/server/internal/pacnexus/config/env.go +++ b/server/internal/pacnexus/config/env.go @@ -8,10 +8,12 @@ import ( var PacSight = struct { Port int + Host string }{} func initPacSightEnv() { - PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 3301) + PacSight.Host = env.GetEnvStringOrDefault("PACSTALL_PACSIGHT_HOST", "localhost") } var PacNexus = struct { diff --git a/server/internal/pacsight/config/env.go b/server/internal/pacsight/config/env.go index ebd0ced1..c8c505ad 100644 --- a/server/internal/pacsight/config/env.go +++ b/server/internal/pacsight/config/env.go @@ -11,15 +11,17 @@ var PacSight = struct { }{} func initPacSightEnv() { - PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) + PacSight.Port = env.GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 3301) } var Repology = struct { RepologyUpdateInterval time.Duration + CachePath string }{} func initRepologyEnv() { Repology.RepologyUpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second + Repology.CachePath = env.GetEnvStringOrDefault("PACSTALL_REPOLOGY_CACHE_PATH", ".") } func Init() { diff --git a/server/internal/pacsight/repology/scheduler.go b/server/internal/pacsight/repology/scheduler.go index f13e7f3e..17b9a4e4 100644 --- a/server/internal/pacsight/repology/scheduler.go +++ b/server/internal/pacsight/repology/scheduler.go @@ -3,8 +3,10 @@ package repology import ( "encoding/json" "os" + "path" "time" + "pacstall.dev/webserver/internal/pacsight/config" "pacstall.dev/webserver/pkg/common/log" "pacstall.dev/webserver/pkg/common/types" ) @@ -32,6 +34,8 @@ func ScheduleRefresh(every time.Duration) { Packages = results if err = cacheValues(Packages); err != nil { log.Error("failed to cache Repology projects: %+v", err) + } else { + log.Debug("repology database cached") } } time.Sleep(every) @@ -44,7 +48,10 @@ func ScheduleRefresh(every time.Duration) { const CACHE_FILE_NAME = "repology_cache.json" func readCache() (types.RepologyApiProjectSearchResponse, error) { - bytes, err := os.ReadFile(CACHE_FILE_NAME) + ensureCacheDirectoryExists() + cacheFile := path.Join(config.Repology.CachePath, CACHE_FILE_NAME) + + bytes, err := os.ReadFile(cacheFile) if err != nil { return nil, err } @@ -58,10 +65,20 @@ func readCache() (types.RepologyApiProjectSearchResponse, error) { } func cacheValues(value types.RepologyApiProjectSearchResponse) error { + log.Debug("caching repology values...") + ensureCacheDirectoryExists() + cacheFile := path.Join(config.Repology.CachePath, CACHE_FILE_NAME) + bytes, err := json.Marshal(value) if err != nil { return err } - return os.WriteFile(CACHE_FILE_NAME, bytes, 0644) + return os.WriteFile(cacheFile, bytes, 0644) +} + +func ensureCacheDirectoryExists() { + if err := os.MkdirAll(config.Repology.CachePath, 0755); err != nil { + log.Fatal("failed to create cache directory: %+v", err) + } } diff --git a/server/internal/pacsight/repology/types/exporter.go b/server/internal/pacsight/repology/types/exporter.go index 2f34933b..3ad60ea2 100644 --- a/server/internal/pacsight/repology/types/exporter.go +++ b/server/internal/pacsight/repology/types/exporter.go @@ -26,7 +26,7 @@ func ExportRepologyDatabase() (types.RepologyApiProjectSearchResponse, error) { time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) } - log.Debug("page %v | cursor at: %v", page, lastProjectName) + log.Trace("page %v | cursor at: %v", page, lastProjectName) var projectPage map[string][]types.RepologyApiProject var err error diff --git a/server/pkg/common/config/vars.go b/server/pkg/common/config/vars.go index d20d83ba..d184c784 100644 --- a/server/pkg/common/config/vars.go +++ b/server/pkg/common/config/vars.go @@ -2,7 +2,10 @@ package config import ( "pacstall.dev/webserver/pkg/common/config/build" + "pacstall.dev/webserver/pkg/common/env" ) var Production = build.Production == "true" || build.Production == "1" var Version = build.Version + +var LogLevel = env.GetEnvStringOrDefault("PACSTALL_LOG_LEVEL", "debug") diff --git a/server/pkg/common/env/env.go b/server/pkg/common/env/env.go deleted file mode 100644 index 6237c973..00000000 --- a/server/pkg/common/env/env.go +++ /dev/null @@ -1,119 +0,0 @@ -package env - -import ( - "time" - - "pacstall.dev/webserver/pkg/common/config/build" -) - -var PacSight = struct { - Port int -}{} - -func initPacSightEnv() { - PacSight.Port = GetEnvIntOrDefault("PACSTALL_PACSIGHT_PORT", 8080) -} - -var PacNexus = struct { - Port int - PublicDir string - Version string - Production bool -}{} - -func initPacNexusEnv() { - PacNexus.Port = GetEnvIntOrDefault("PACSTALL_PACNEXUS_PORT", 3300) - PacNexus.PublicDir = GetEnvStringOrDefault("PACSTALL_PACNEXUS_PUBLIC_DIR", "./public") - PacNexus.Version = build.Version - PacNexus.Production = toBool(build.Production) -} - -// Configuration for the discord integration -var Discord = struct { - Token string - ChannelID string - Enabled bool - Tags string -}{} - -func initDiscordEnv() { - Discord.Enabled = GetEnvBoolOrDefault("PACSTALL_DISCORD_ENABLED", false) - - if !Discord.Enabled { - return - } - - Discord.Token = GetEnvString("PACSTALL_DISCORD_TOKEN") - Discord.ChannelID = GetEnvString("PACSTALL_DISCORD_CHANNEL_ID") - Discord.Tags = GetEnvString("PACSTALL_DISCORD_TAGS") -} - -var PacstallPrograms = struct { - Branch string - UpdateInterval time.Duration - TempDir string - MaxOpenFiles int - GitURL string - GitClonePath string -}{} - -func initPacstallProgramsEnv() { - PacstallPrograms.Branch = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_BRANCH", "master") - PacstallPrograms.UpdateInterval = time.Duration(GetEnvIntOrDefault("PACSTALL_PROGRAMS_UPDATE_INTERVAL", 15*60)) * time.Second - PacstallPrograms.TempDir = GetEnvStringOrDefault("PACSTALL_PROGRAMS_TEMP_DIR", "./tmp") - PacstallPrograms.MaxOpenFiles = GetEnvIntOrDefault("PACSTALL_PROGRAMS_MAX_OPEN_FILES", 100) - PacstallPrograms.GitURL = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_URL", "https://github.com/pacstall/pacstall-programs.git") - PacstallPrograms.GitClonePath = GetEnvStringOrDefault("PACSTALL_PROGRAMS_GIT_CLONE_PATH", "./programs") -} - -// Configuration for the database -var Database = struct { - Host string - Port int - User string - Password string - Name string -}{} - -func initDatabaseEnv() { - Database.Host = GetEnvStringOrDefault("PACSTALL_DATABASE_HOST", "localhost") - Database.Port = GetEnvIntOrDefault("PACSTALL_DATABASE_PORT", 3306) - Database.User = GetEnvStringOrDefault("PACSTALL_DATABASE_USER", "root") - Database.Password = GetEnvStringOrDefault("PACSTALL_DATABASE_PASSWORD", "changeme") - Database.Name = GetEnvStringOrDefault("PACSTALL_DATABASE_NAME", "pacstall") -} - -// Configuration for the Matomo API -var Matomo = struct { - Enabled bool -}{} - -func initMatomoEnv() { - Matomo.Enabled = GetEnvBoolOrDefault("PACSTALL_MATOMO_ENABLED", false) -} - -// Configuration for the Repology API -var Repology = struct { - Enabled bool - RepologyUpdateInterval time.Duration -}{} - -func initRepologyEnv() { - Repology.Enabled = GetEnvBoolOrDefault("PACSTALL_REPOLOGY_ENABLED", false) - - if !Repology.Enabled { - return - } - - Repology.RepologyUpdateInterval = time.Duration(GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second -} - -func Init() { - initPacNexusEnv() - initPacSightEnv() - initDatabaseEnv() - initDiscordEnv() - initMatomoEnv() - initPacstallProgramsEnv() - initRepologyEnv() -} diff --git a/server/pkg/common/log/lib.go b/server/pkg/common/log/lib.go index b6bc6e89..ba9cf1a0 100644 --- a/server/pkg/common/log/lib.go +++ b/server/pkg/common/log/lib.go @@ -4,38 +4,49 @@ import ( "fmt" glog "log" "os" + "strings" "github.com/fatih/color" + "pacstall.dev/webserver/pkg/common/config" ) var ( logInfo = color.CyanString("INFO") logError = color.RedString("ERROR") logDebug = color.GreenString("DEBUG") + logTrace = color.HiGreenString("TRACE") logFatal = color.New(color.BgHiRed, color.FgBlack).Sprintf("FATAL") logWarn = color.YellowString("WARN") ) type tLogLevel uint8 -var Level = struct { - Info tLogLevel - Error tLogLevel - Debug tLogLevel - Fatal tLogLevel - Warn tLogLevel -}{ - Debug: 0, - Info: 1, - Warn: 2, - Error: 3, - Fatal: 4, +const ( + _LEVEL_TRACE tLogLevel = iota + _LEVEL_DEBUG + _LEVEL_INFO + _LEVEL_WARN + _LEVEL_ERROR + _LEVEL_FATAL +) + +var logLevels = map[string]tLogLevel{ + "TRACE": _LEVEL_TRACE, + "DEBUG": _LEVEL_DEBUG, + "INFO": _LEVEL_INFO, + "WARN": _LEVEL_WARN, + "ERROR": _LEVEL_ERROR, + "FATAL": _LEVEL_FATAL, } -var logLevel = Level.Debug +func getLogLevel() tLogLevel { + l, ok := logLevels[strings.ToUpper(config.LogLevel)] + if !ok { + doLog(logWarn, "unknown log level '%s'. defaulting to INFO", config.LogLevel) + return _LEVEL_INFO + } -func SetLogLevel(level tLogLevel) { - logLevel = level + return l } var logger = glog.New(os.Stdout, "", glog.Ldate|glog.Ltime) @@ -51,7 +62,7 @@ func doLog(level, message string, args ...any) { } func Info(message string, args ...any) { - if logLevel > Level.Info { + if getLogLevel() > _LEVEL_INFO { return } @@ -59,7 +70,7 @@ func Info(message string, args ...any) { } func Error(message string, args ...any) { - if logLevel > Level.Error { + if getLogLevel() > _LEVEL_ERROR { return } @@ -67,7 +78,7 @@ func Error(message string, args ...any) { } func Fatal(message string, args ...any) { - if logLevel > Level.Fatal { + if getLogLevel() > _LEVEL_FATAL { return } @@ -75,7 +86,7 @@ func Fatal(message string, args ...any) { } func Warn(message string, args ...any) { - if logLevel > Level.Warn { + if getLogLevel() > _LEVEL_WARN { return } @@ -83,9 +94,17 @@ func Warn(message string, args ...any) { } func Debug(message string, args ...any) { - if logLevel > Level.Debug { + if getLogLevel() > _LEVEL_DEBUG { return } doLog(logDebug, message, args...) } + +func Trace(message string, args ...any) { + if getLogLevel() > _LEVEL_TRACE { + return + } + + doLog(logTrace, message, args...) +} From a708d3a5298f6de30f6aebdd8fd1077c8c1357ec Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Wed, 25 Sep 2024 21:22:11 +0300 Subject: [PATCH 15/17] chore: update infra and repology caching --- .github/workflows/development.yml | 25 ++-- .github/workflows/production.yml | 25 ++-- docker-compose.dev.yml | 2 - .../pac/parser/pacsh/internal/git_version.go | 2 +- .../pacnexus/types/pac/parser/parse.go | 4 +- server/internal/pacsight/config/env.go | 4 +- .../internal/pacsight/repology/scheduler.go | 112 ++++++++++++++---- .../pacsight/repology/types/exporter.go | 99 ++++++++++------ .../common}/parallelism/batch/run.go | 0 .../common}/parallelism/batch/run_test.go | 0 .../common}/parallelism/channels/exhaust.go | 0 .../common}/parallelism/channels/to_slice.go | 0 .../parallelism/channels/to_slice_test.go | 0 .../common}/parallelism/timeout/timeout.go | 0 14 files changed, 189 insertions(+), 84 deletions(-) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/batch/run.go (100%) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/batch/run_test.go (100%) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/channels/exhaust.go (100%) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/channels/to_slice.go (100%) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/channels/to_slice_test.go (100%) rename server/{internal/pacnexus/types/pac/parser => pkg/common}/parallelism/timeout/timeout.go (100%) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index c04895b8..63c26673 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -24,13 +24,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.PUBLISH_PACKAGE_PAT }} - - name: Build Docker Images + - name: Build PacNexus Docker Images run: | - docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t webserver . - docker tag webserver "ghcr.io/pacstall/webserver:$SHORT_SHA" - docker tag webserver ghcr.io/pacstall/webserver:development + docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t pacnexus pacnexus.Dockerfile + docker tag pacnexus "ghcr.io/pacstall/pacnexus:$SHORT_SHA" + docker tag pacnexus ghcr.io/pacstall/pacnexus:development - - name: Push Images + - name: Push PacNexus Images run: | - docker push "ghcr.io/pacstall/webserver:$SHORT_SHA" - docker push ghcr.io/pacstall/webserver:development \ No newline at end of file + docker push "ghcr.io/pacstall/pacnexus:$SHORT_SHA" + docker push ghcr.io/pacstall/pacnexus:development + + - name: Build PacSight Docker Images + run: | + docker build --build-arg version="${SHORT_SHA}-dev" --no-cache -t pacsight pacsight.Dockerfile + docker tag pacsight "ghcr.io/pacstall/pacsight:$SHORT_SHA" + docker tag pacsight ghcr.io/pacstall/pacsight:development + + - name: Push PacSight Images + run: | + docker push "ghcr.io/pacstall/pacsight:$SHORT_SHA" + docker push ghcr.io/pacstall/pacsight:development \ No newline at end of file diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 3463aa4e..bddaca61 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -22,13 +22,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.PUBLISH_PACKAGE_PAT }} - - name: Build Docker image + - name: Build PacNexus Docker image run: | - docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t webserver . - docker tag webserver "ghcr.io/pacstall/webserver:${{ github.event.inputs.version }}" - docker tag webserver ghcr.io/pacstall/webserver:latest + docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t pacnexus pacnexus.Dockerfile + docker tag pacnexus "ghcr.io/pacstall/pacnexus:${{ github.event.inputs.version }}" + docker tag pacnexus ghcr.io/pacstall/pacnexus:latest - - name: Push Image + - name: Build PacSight Docker image run: | - docker push "ghcr.io/pacstall/webserver:${{ github.event.inputs.version }}" - docker push ghcr.io/pacstall/webserver:latest + docker build --build-arg VITE_VERSION="${{ github.event.inputs.version }}" --no-cache -t pacsight pacsight.Dockerfile + docker tag pacsight "ghcr.io/pacstall/pacsight:${{ github.event.inputs.version }}" + docker tag pacsight ghcr.io/pacstall/pacsight:latest + + - name: Push PacNexus Image + run: | + docker push "ghcr.io/pacstall/pacnexus:${{ github.event.inputs.version }}" + docker push ghcr.io/pacstall/pacnexus:latest + + - name: Push PacSight Image + run: | + docker push "ghcr.io/pacstall/pacsight:${{ github.event.inputs.version }}" + docker push ghcr.io/pacstall/pacsight:latest diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7325a554..3743c1a7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: '3.2' - services: mariadb: diff --git a/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go index 7c63f746..daeba4b3 100644 --- a/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go +++ b/server/internal/pacnexus/types/pac/parser/pacsh/internal/git_version.go @@ -7,7 +7,7 @@ import ( "github.com/joomcode/errorx" "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" - "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/timeout" + "pacstall.dev/webserver/pkg/common/parallelism/timeout" ) type GitSourceInfo struct { diff --git a/server/internal/pacnexus/types/pac/parser/parse.go b/server/internal/pacnexus/types/pac/parser/parse.go index 08535b75..e7461cc6 100644 --- a/server/internal/pacnexus/types/pac/parser/parse.go +++ b/server/internal/pacnexus/types/pac/parser/parse.go @@ -12,11 +12,11 @@ import ( "pacstall.dev/webserver/internal/pacnexus/types/pac/pacstore" "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/git" "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/pacsh" - "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/batch" - "pacstall.dev/webserver/internal/pacnexus/types/pac/parser/parallelism/channels" "pacstall.dev/webserver/pkg/common/array" "pacstall.dev/webserver/pkg/common/log" "pacstall.dev/webserver/pkg/common/pacsight" + "pacstall.dev/webserver/pkg/common/parallelism/batch" + "pacstall.dev/webserver/pkg/common/parallelism/channels" "pacstall.dev/webserver/pkg/common/types" ) diff --git a/server/internal/pacsight/config/env.go b/server/internal/pacsight/config/env.go index c8c505ad..f8083ae0 100644 --- a/server/internal/pacsight/config/env.go +++ b/server/internal/pacsight/config/env.go @@ -17,11 +17,13 @@ func initPacSightEnv() { var Repology = struct { RepologyUpdateInterval time.Duration CachePath string + MaxOpenFiles int }{} func initRepologyEnv() { Repology.RepologyUpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second - Repology.CachePath = env.GetEnvStringOrDefault("PACSTALL_REPOLOGY_CACHE_PATH", ".") + Repology.CachePath = env.GetEnvStringOrDefault("PACSTALL_REPOLOGY_CACHE_PATH", "./repology_cache") + Repology.MaxOpenFiles = env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_MAX_OPEN_FILES", 10) } func Init() { diff --git a/server/internal/pacsight/repology/scheduler.go b/server/internal/pacsight/repology/scheduler.go index 17b9a4e4..be9a98f2 100644 --- a/server/internal/pacsight/repology/scheduler.go +++ b/server/internal/pacsight/repology/scheduler.go @@ -1,12 +1,16 @@ package repology import ( - "encoding/json" + "bufio" + "encoding/gob" + "io/fs" "os" "path" "time" + "github.com/joomcode/errorx" "pacstall.dev/webserver/internal/pacsight/config" + "pacstall.dev/webserver/pkg/common/array" "pacstall.dev/webserver/pkg/common/log" "pacstall.dev/webserver/pkg/common/types" ) @@ -19,25 +23,38 @@ func ScheduleRefresh(every time.Duration) { if err != nil { log.Warn("no cache found. this is normal on clean runs: %+v", err) } else { - log.Info("repology cache read successfully") + log.Info("repology cache read successfully. %d projects found", len(cache)) Packages = cache } go func() { for { log.Info("refreshing Repology database...") - results, err := ExportRepologyDatabase() - if err != nil { - log.Error("failed to export Repology projects: %+v", err) - } else { - log.Info("repology database refreshed successfully") - Packages = results - if err = cacheValues(Packages); err != nil { - log.Error("failed to cache Repology projects: %+v", err) - } else { - log.Debug("repology database cached") + out := types.RepologyApiProjectSearchResponse{} + resultsChan, errChan := ExportRepologyDatabase() + + chanLoop: + for { + select { + case pair, ok := <-resultsChan: + if !ok { + log.Info("repology database refreshed successfully. %d projects found", len(out)) + break chanLoop + } + + out[pair.ProjectName] = pair.Projects + + if err := cachePair(pair.ProjectName, pair.Projects); err != nil { + log.Error("failed to cache Repology projects: %+v", err) + } else { + log.Trace("cached repology project '%s'", pair.ProjectName) + } + case err := <-errChan: + log.Error("failed to export Repology projects: %+v", err) + break chanLoop } } + time.Sleep(every) } }() @@ -45,36 +62,81 @@ func ScheduleRefresh(every time.Duration) { log.Info("scheduled repology refresh every %v", every) } -const CACHE_FILE_NAME = "repology_cache.json" - func readCache() (types.RepologyApiProjectSearchResponse, error) { ensureCacheDirectoryExists() - cacheFile := path.Join(config.Repology.CachePath, CACHE_FILE_NAME) - bytes, err := os.ReadFile(cacheFile) - if err != nil { + var err error + var dirEntries []fs.DirEntry + if dirEntries, err = os.ReadDir(config.Repology.CachePath); err != nil { return nil, err } - var cache types.RepologyApiProjectSearchResponse = make(map[string][]types.RepologyApiProject) - if err = json.Unmarshal(bytes, &cache); err != nil { - return nil, err + dirEntries = array.Filter(dirEntries, func(entry *array.Iterator[fs.DirEntry]) bool { + return !entry.Value.IsDir() + }) + + if len(dirEntries) == 0 { + return nil, errorx.DataUnavailable.New("no cache files found") + } + + files := array.SwitchMap(dirEntries, func(entry *array.Iterator[fs.DirEntry]) string { + return entry.Value.Name() + }) + + cache := types.RepologyApiProjectSearchResponse{} + + for _, fileName := range files { + cacheFilePath := path.Join(config.Repology.CachePath, fileName) + file, err := os.Open(cacheFilePath) + if err != nil { + return nil, err + } + + defer file.Close() + + reader := bufio.NewReader(file) + decoder := gob.NewDecoder(reader) + + projects := []types.RepologyApiProject{} + if err = decoder.Decode(&projects); err != nil { + return nil, err + } + + projectName := fileName[:len(fileName)-4] // remove .gob extension + + cache[projectName] = projects } return cache, nil } -func cacheValues(value types.RepologyApiProjectSearchResponse) error { - log.Debug("caching repology values...") +func cachePair(projectName string, projects []types.RepologyApiProject) error { ensureCacheDirectoryExists() - cacheFile := path.Join(config.Repology.CachePath, CACHE_FILE_NAME) + fileName := projectName + ".gob" + cacheFilePath := path.Join(config.Repology.CachePath, fileName) + + // delete existing cache file if it exists + os.RemoveAll(cacheFilePath) - bytes, err := json.Marshal(value) + file, err := os.Create(cacheFilePath) if err != nil { return err } - return os.WriteFile(cacheFile, bytes, 0644) + defer file.Close() + + writer := bufio.NewWriter(file) + encoder := gob.NewEncoder(writer) + + if err = encoder.Encode(projects); err != nil { + return err + } + + if err = writer.Flush(); err != nil { + return err + } + + return nil } func ensureCacheDirectoryExists() { diff --git a/server/internal/pacsight/repology/types/exporter.go b/server/internal/pacsight/repology/types/exporter.go index 3ad60ea2..c53e15c7 100644 --- a/server/internal/pacsight/repology/types/exporter.go +++ b/server/internal/pacsight/repology/types/exporter.go @@ -14,61 +14,82 @@ import ( const RETRY_COUNT = 10 const REPOLOGY_PROJECT_FETCH_THROTTLE = time.Second -func ExportRepologyDatabase() (types.RepologyApiProjectSearchResponse, error) { - page := 1 - lastProjectName := "" - - lastRepoFetch := time.Now() - var repologyProjectsMap = make(types.RepologyApiProjectSearchResponse) +type RepologyProjectPair struct { + ProjectName string + Projects []types.RepologyApiProject +} - for { - if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { - time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) - } +func ExportRepologyDatabase() (<-chan RepologyProjectPair, <-chan error) { + out := make(chan RepologyProjectPair) + errs := make(chan error) - log.Trace("page %v | cursor at: %v", page, lastProjectName) + go func() { + page := 1 + lastProjectName := "" - var projectPage map[string][]types.RepologyApiProject - var err error + lastRepoFetch := time.Now() + var repologyProjectsMap = make(types.RepologyApiProjectSearchResponse) - retry: - for i := 1; i <= RETRY_COUNT; i += 1 { - projectPage, err = getProjectSearch(lastProjectName) - if err == nil { - break retry + for { + if time.Since(lastRepoFetch) < REPOLOGY_PROJECT_FETCH_THROTTLE { + time.Sleep(REPOLOGY_PROJECT_FETCH_THROTTLE - time.Since(lastRepoFetch)) } - retryDelay := time.Duration(math.Pow(float64(i), 2)) * REPOLOGY_PROJECT_FETCH_THROTTLE - log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) - time.Sleep(retryDelay) - } + log.Trace("page %v | cursor at: %v", page, lastProjectName) - if err != nil { - return nil, errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) - } + var projectPage map[string][]types.RepologyApiProject + var err error + + retry: + for i := 1; i <= RETRY_COUNT; i += 1 { + projectPage, err = getProjectSearch(lastProjectName) + if err == nil { + break retry + } - lastRepoFetch = time.Now() + retryDelay := time.Duration(math.Pow(float64(i), 2)) * REPOLOGY_PROJECT_FETCH_THROTTLE + log.Debug("failed to fetch repology project page '%s'. retrying in %v", lastProjectName, retryDelay) + time.Sleep(retryDelay) + } - shouldStop := false - for projectName, apiProjectProvider := range projectPage { - if _, ok := repologyProjectsMap[projectName]; !ok { - repologyProjectsMap[projectName] = []types.RepologyApiProject{} + if err != nil { + errs <- errorx.Decorate(err, "failed to fetch repology project page '%s'", lastProjectName) + close(out) + close(errs) + return } - repologyProjectsMap[projectName] = append(repologyProjectsMap[projectName], apiProjectProvider...) + lastRepoFetch = time.Now() - shouldStop = projectName == lastProjectName - lastProjectName = identityOrSkipProject(projectName) - } + shouldStop := false + for projectName, apiProjectProvider := range projectPage { + if _, ok := repologyProjectsMap[projectName]; !ok { + repologyProjectsMap[projectName] = []types.RepologyApiProject{} + } - if shouldStop { - break + repologyProjectsMap[projectName] = append(repologyProjectsMap[projectName], apiProjectProvider...) + + out <- RepologyProjectPair{ + ProjectName: projectName, + Projects: apiProjectProvider, + } + + shouldStop = projectName == lastProjectName + lastProjectName = identityOrSkipProject(projectName) + } + + if shouldStop { + break + } + + page += 1 } - page += 1 - } + close(errs) + close(out) + }() - return repologyProjectsMap, nil + return out, errs } var projectNamesToSkipToNextCussor = map[string]string{ diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/batch/run.go b/server/pkg/common/parallelism/batch/run.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/batch/run.go rename to server/pkg/common/parallelism/batch/run.go diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/batch/run_test.go b/server/pkg/common/parallelism/batch/run_test.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/batch/run_test.go rename to server/pkg/common/parallelism/batch/run_test.go diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/channels/exhaust.go b/server/pkg/common/parallelism/channels/exhaust.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/channels/exhaust.go rename to server/pkg/common/parallelism/channels/exhaust.go diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice.go b/server/pkg/common/parallelism/channels/to_slice.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice.go rename to server/pkg/common/parallelism/channels/to_slice.go diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice_test.go b/server/pkg/common/parallelism/channels/to_slice_test.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/channels/to_slice_test.go rename to server/pkg/common/parallelism/channels/to_slice_test.go diff --git a/server/internal/pacnexus/types/pac/parser/parallelism/timeout/timeout.go b/server/pkg/common/parallelism/timeout/timeout.go similarity index 100% rename from server/internal/pacnexus/types/pac/parser/parallelism/timeout/timeout.go rename to server/pkg/common/parallelism/timeout/timeout.go From be33c9fad96dfb91a8a125d4511a9d7bd138f434 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Wed, 25 Sep 2024 21:23:23 +0300 Subject: [PATCH 16/17] chore: remove unused env var --- server/internal/pacsight/config/env.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/internal/pacsight/config/env.go b/server/internal/pacsight/config/env.go index f8083ae0..9e3f2ddb 100644 --- a/server/internal/pacsight/config/env.go +++ b/server/internal/pacsight/config/env.go @@ -23,7 +23,6 @@ var Repology = struct { func initRepologyEnv() { Repology.RepologyUpdateInterval = time.Duration(env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_UPDATE_INTERVAL", 60*60*6)) * time.Second Repology.CachePath = env.GetEnvStringOrDefault("PACSTALL_REPOLOGY_CACHE_PATH", "./repology_cache") - Repology.MaxOpenFiles = env.GetEnvIntOrDefault("PACSTALL_REPOLOGY_MAX_OPEN_FILES", 10) } func Init() { From bf83aad243a7e60e70ce645005b609a94a3b8285 Mon Sep 17 00:00:00 2001 From: Paul Cosma Date: Thu, 26 Sep 2024 12:18:16 +0300 Subject: [PATCH 17/17] refactor: Remove unnecessary log statements and improve error handling in repology scheduler --- server/internal/pacsight/repology/scheduler.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/internal/pacsight/repology/scheduler.go b/server/internal/pacsight/repology/scheduler.go index be9a98f2..1f670305 100644 --- a/server/internal/pacsight/repology/scheduler.go +++ b/server/internal/pacsight/repology/scheduler.go @@ -38,7 +38,6 @@ func ScheduleRefresh(every time.Duration) { select { case pair, ok := <-resultsChan: if !ok { - log.Info("repology database refreshed successfully. %d projects found", len(out)) break chanLoop } @@ -50,11 +49,15 @@ func ScheduleRefresh(every time.Duration) { log.Trace("cached repology project '%s'", pair.ProjectName) } case err := <-errChan: - log.Error("failed to export Repology projects: %+v", err) - break chanLoop + if err != nil { + log.Error("failed to export Repology projects: %+v", err) + break chanLoop + } } } + log.Info("repology database refreshed successfully. %d projects found", len(out)) + Packages = out time.Sleep(every) } }()