From e61d2e9a30cb289b6479ab358671c38f370fedcf Mon Sep 17 00:00:00 2001 From: Ctrl-S Date: Fri, 5 Feb 2021 12:55:10 +0800 Subject: [PATCH 1/4] Attempting to share the version I've made by tweaking what suika published. --- .gitignore | 12 ++++++ Torako.toml | 28 ++++++------- ayase.config.toml | 4 ++ docker-compose.yml | 28 +++++++++++-- mkdirs.sh | 18 +++++++++ readme.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 172 insertions(+), 17 deletions(-) create mode 100644 .gitignore create mode 100755 mkdirs.sh create mode 100644 readme.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a44f48 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +## .gitignore + +## Private +git_ignored/ + +## Used by containers +seaweedfs/ +mysql/ + +## Clutter +.vscode/ +*.~* \ No newline at end of file diff --git a/Torako.toml b/Torako.toml index 905fdb5..f9be753 100644 --- a/Torako.toml +++ b/Torako.toml @@ -64,21 +64,21 @@ request_timeout = "30m" #download_media = true ## List the boards that should be archived -[boards.a] -[boards.c] -[boards.d] -[boards.e] -[boards.f] -[boards.fit] +#[boards.a] +#[boards.c] +#[boards.d] +#[boards.e] +#[boards.f] +#[boards.fit] [boards.g] -[boards.gif] -[boards.h] -[boards.ic] -[boards.jp] -[boards.u] -[boards.w] -[boards.wg] -[boards.vt] +#[boards.gif] +#[boards.h] +#[boards.ic] +#[boards.jp] +#[boards.u] +#[boards.w] +#[boards.wg] +#[boards.vt] ## Any of the above options can be provided ## here for board specific options diff --git a/ayase.config.toml b/ayase.config.toml index 844a49e..9547637 100644 --- a/ayase.config.toml +++ b/ayase.config.toml @@ -100,6 +100,10 @@ name = "Retro Games" shortname = "wsg" name = "Worksafe GIF" +[[archives]] +shortname = "g" +name = "Computer nerd shit" + ## List of non-archive boards to display. ## Boards currently are no different from archives, although it may be used for text ## boards in the future. diff --git a/docker-compose.yml b/docker-compose.yml index 9c122f9..0f17ed8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: ports: - 127:0.0.1:8080:8080 # - 127:0.0.1:18080:18080 - command: 'volume -mserver="seaweed-master:9333" -port=8080 -dataCenter=asagi -rack=hdd' + command: 'volume -mserver="seaweed-master:9333" -port=8080 -dataCenter=asagi -rack=hdd -max=32' depends_on: seaweed-master: condition: service_started @@ -41,11 +41,15 @@ services: - 7865:7865 - 127:0.0.1:8888:8888 # - 127:0.0.1:18888:18888 + environment: + WEED_LEVELDB2_ENABLED: 'true' + WEED_LEVELDB2_DIR: '/filerldb2' command: 'filer -master="seaweed-master:9333" -disableDirListing -port.readonly=7865' tty: true stdin_open: true volumes: - ./seaweedfs/filer:/data + - ./seaweedfs/filer/filerldb2:/filerldb2 depends_on: seaweed-master: condition: service_started @@ -98,8 +102,8 @@ services: cap_add: - SYS_NICE environment: - - MYSQL_ROOT_PASSWORD='bunbuncha' - - MYSQL_DATABASE=asagi + MYSQL_ROOT_PASSWORD: 'bunbuncha' + MYSQL_DATABASE: 'asagi' ports: - 127:0.0.1:3306:3306 volumes: @@ -112,10 +116,28 @@ services: 4chan-scrape-db: condition: service_started restart: always + networks: + - foolstack + ports: + - 127:0.0.1:29080:8000 command: ["uvicorn","view.asagi:app", "--host", "0.0.0.0"] volumes: - ./ayase.config.toml:/opt/ayase/config.toml + # prepare-dirs: + # image: alpine + # container_name: prepare-dirs + # restart: 'no' + # # command: ["mkdir", "-vp", + # # "/o/seaweedfs/master/data", + # # "/o/seaweedfs/seaweed-volume-1", + # # "/o/seaweedfs/filer", + # # "/o/seaweedfs/filer/filerldb2"] + # command: '/o/mkdirs.sh' + # volumes: + # ## /o/ = Outside + # - .:/o/ + networks: foolstack: name: foolstack \ No newline at end of file diff --git a/mkdirs.sh b/mkdirs.sh new file mode 100755 index 0000000..77ce413 --- /dev/null +++ b/mkdirs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +## mkdirs.sh +## Ensure required dirs exist. + +# cd /o/ + +## seaweed-master +mkdir -vp ./seaweedfs/master/data +chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/master/ ./seaweedfs/master/data + +## seaweed-volume-1 +mkdir -vp ./seaweedfs/seaweed-volume-1 +chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/seaweed-volume-1 + +## seaweed-filer +mkdir -vp ./seaweedfs/filer/filerldb2 +chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/filer ./seaweedfs/filer/filerldb2 + diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..9fbc138 --- /dev/null +++ b/readme.md @@ -0,0 +1,99 @@ +# Foolfstack - torako + seaweedfs + ayase +Quick guide to using this stuff. +Non-local access is outside the current scope. + +## Install preprequisites +* Install docker + +https://docs.docker.com/engine/install/fedora/ +``` +## Update system packages +$ sudo dnf -y update +$ sudo dnf -y install dnf-plugins-core + +## Add docker repository: +$ sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo +$ sudo dnf makecache + +## Install docker packages: +$ sudo dnf install docker-ce docker-ce-cli containerd.io + +## Start docker service: +$ sudo systemctl enable --now docker +$ systemctl status docker + +## Add current user to docker group: +$ sudo usermod -aG docker $(whoami) + +## "login" to docker group in an existing terminal session: +$ newgrp docker +``` + +* Install docker-compose +``` +$ sudo dnf install docker-compose +``` + + +## Prep +Seaweedfs does not implicaitly create many of its dirs. +This script makes them if they do not exist and assigns permissions permissive enough to make sure seaweedfs can use them. +``` +$ bash mkdirs.sh +``` + + +## Running: +Start/create containers in the background: +``` +$ docker-compose -f "docker-compose.yml" up -d +``` + +Stop/remove containers: +``` +$ docker-compose -f "docker-compose.yml" down +``` + + +### Viewing logs +``` +$ docker-compose -f "docker-compose.yml" logs +$ docker-compose -f "docker-compose.yml" logs ayase +``` + +### Running commands inside containers manually: +``` +$ docker exec -it 4chan-scrape-db sh +container$ mysql -uroot -pbunbuncha +container.mysql> SELECT host, user FROM mysql.user; +``` + +Installing packages in container for diagnostics: +``` +apt-get update && apt-get install -y traceroute curl iputils +``` + + +## Ayase website +In a web browser, open this address to view the ayase instance provided by the container "ayase": + +`http://127:0.0.1:29080/` + + +## DB Export +* Note: The method demonstrated does not strip secrets such as deletion passwords or IP addresses from the generated file. + +Backup the entire database: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables --all-databases | gzip > "4chan-scrape-db.all-databases.t`date -u +%s`.sql.gz" +``` + +Export a single board: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables "asagi" "g" "g_images" | gzip > "4chan-scrape-db.g.t`date -u +%s`.sql.gz" +``` + +Export multiple boards: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables "asagi" "g" "g_images" "co" "co_images" | gzip > "4chan-scrape-db.my_boards_export.t`date -u +%s`.sql.gz" +``` From 98336481a42f785e4f18be9d4e976228a5ff30e8 Mon Sep 17 00:00:00 2001 From: Ctrl-S Date: Thu, 11 Feb 2021 17:08:24 +0800 Subject: [PATCH 2/4] WIP trying to sync with suika origin --- .gitignore | 1 + Torako.toml | 12 ++++++------ ayase.config.toml | 2 +- docker-compose.yml | 3 ++- misc_notes.md | 12 ++++++++++++ 5 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 misc_notes.md diff --git a/.gitignore b/.gitignore index 2a44f48..9692323 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ mysql/ ## Clutter .vscode/ +*.code-workspace *.~* \ No newline at end of file diff --git a/Torako.toml b/Torako.toml index f9be753..1aebe15 100644 --- a/Torako.toml +++ b/Torako.toml @@ -10,7 +10,7 @@ api_addr = "127.0.0.1:2377" ## this rate limit only affects downloading the JSON threads/posts ## and will ensure api requests are throttled appropirately. ## If this option is not specified, there will be no rate limiting -#rate_limit = 10 +rate_limit = 2 ## Global board thread concurrency. This is a limit on the amount of ## of threads that will be retrieved and buffered at once. If this limit @@ -49,7 +49,7 @@ request_timeout = "30m" #host = "a.4cdn.org" ## Default refresh rate for boards (Default: 10s) -#refresh_rate = "10s" +refresh_rate = "30s" ## Default deleted page threshold. If threads ## are removed before this threshold is reached @@ -58,10 +58,10 @@ request_timeout = "30m" #deleted_page_threshold = 8 ## Download thumbnails (Default: true) -#download_thumbs = true +download_thumbs = true ## Download media/images (Default: true) -#download_media = true +download_media = true ## List the boards that should be archived #[boards.a] @@ -149,7 +149,7 @@ sha_dir_structure = false ## concurrent downloads will ont exceed this number. ## Tune this number according to your memory and bandwidth preferences ## (Default: 128) -concurrent_downloads = 10 +concurrent_downloads = 16 ## Setting this to true means that the media download queue will backpressure ## the system, and once all concurrent_download slots are full, no more new @@ -289,7 +289,7 @@ compute_stats = true ## like MySQL, UTC mode must be set explictly. This can be safely disabled ## if your database is already in UTC. ## (Default: true) -#sql_set_utc = true +sql_set_utc = true [backend.asagi_pg_search] disabled = true diff --git a/ayase.config.toml b/ayase.config.toml index 9547637..19e69d0 100644 --- a/ayase.config.toml +++ b/ayase.config.toml @@ -162,7 +162,7 @@ reports = false ## The following is the default for ayase. ## You may also specify a full URL. [image_location] -image = "http://127.0.0.1:7865/buckets/asagi-images/{board_name}/image" +image = "http://127.0.0.1:7865/buckets/asagi-thumbs/{board_name}/image" thumb = "http://127.0.0.1:7865/buckets/asagi-thumbs/{board_name}/thumb" ## Torako sha256 folder structure. diff --git a/docker-compose.yml b/docker-compose.yml index 0f17ed8..4b182a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,8 @@ services: environment: WEED_LEVELDB2_ENABLED: 'true' WEED_LEVELDB2_DIR: '/filerldb2' - command: 'filer -master="seaweed-master:9333" -disableDirListing -port.readonly=7865' + #command: 'filer -master="seaweed-master:9333" -disableDirListing -port.readonly=7865' + command: 'filer -master="seaweed-master:9333" -port.readonly=7865' tty: true stdin_open: true volumes: diff --git a/misc_notes.md b/misc_notes.md new file mode 100644 index 0000000..4db7bce --- /dev/null +++ b/misc_notes.md @@ -0,0 +1,12 @@ + +# 2020-02-11 suika latest version in branch "ayase" + + +## Create buckets by hand: +$ docker exec -it seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 +> s3.bucket.create -name asagi-thumbs +> s3.bucket.create -name asagi-images + + + + From ea8f2a72762e5d36dba94ebff19cc973c3ca9fa6 Mon Sep 17 00:00:00 2001 From: Ctrl-S Date: Thu, 11 Feb 2021 18:33:42 +0800 Subject: [PATCH 3/4] Added script to handle startup. Mimized active boards down to just /g/ with both thumbs+images. Ayase bucket config fix. --- README.md | 7 +-- Torako.toml | 2 +- ayase.config.toml | 118 ++++++++++++++++++++++----------------------- docker-compose.yml | 25 ++-------- mkdir.sh | 17 ------- startup.sh | 32 ++++++++++++ 6 files changed, 101 insertions(+), 100 deletions(-) delete mode 100644 mkdir.sh create mode 100644 startup.sh diff --git a/README.md b/README.md index 65d451b..6677cf0 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ $ docker-compose -f "docker-compose.yml" up -d Create the buckets where the files will be located (this creates two different collections, basically own volumes different from others): ``` -docker exec seaweed-master -it /usr/bin/weed -master=seaweed-master:9333 -filer=seaweed-filer:8888 -s3.bucket.create -name asagi-thumbs -s3.bucket.create -name asagi-images +$ docker exec -it seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 +> s3.bucket.create -name asagi-thumbs +> s3.bucket.create -name asagi-images + ctrl+d/ctrl+c/exit ``` diff --git a/Torako.toml b/Torako.toml index 0148bc3..9a7c437 100644 --- a/Torako.toml +++ b/Torako.toml @@ -112,7 +112,7 @@ old_dir_structure = false ## Sha256 Directory Structure. Images will be stored in accordance to the ## sha256 hash of the image. (Default: false) -sha_dir_structure = false +sha_dir_structure = true ## If we fail to save a thread or post to MySQL for whatever ## reason, we should consider this as a fatal error and diff --git a/ayase.config.toml b/ayase.config.toml index 19e69d0..1af5ca8 100644 --- a/ayase.config.toml +++ b/ayase.config.toml @@ -24,81 +24,81 @@ authentication_type = "oauth2" ## List of archived boards to display. -[[archives]] -shortname = "a" -name = "Anime & Manga" +#[[archives]] +#shortname = "a" +#name = "Anime & Manga" -[[archives]] -shortname = "aco" -name = "Adult Cartoons" +#[[archives]] +#shortname = "aco" +#name = "Adult Cartoons" -[[archives]] -shortname = "an" -name = "Animals & Nature" +#[[archives]] +#shortname = "an" +#name = "Animals & Nature" -[[archives]] -shortname = "c" -name = "Cute" +#[[archives]] +#shortname = "c" +#name = "Cute" -[[archives]] -shortname = "co" -name = "Comics & Cartoons" +#[[archives]] +#shortname = "co" +#name = "Comics & Cartoons" -[[archives]] -shortname = "d" -name = "Hentai/Alternative" +#[[archives]] +#shortname = "d" +#name = "Hentai/Alternative" -[[archives]] -shortname = "fit" -name = "Fitness" +#[[archives]] +#shortname = "fit" +#name = "Fitness" -[[archives]] -shortname = "his" -name = "History & Humanities" +#[[archives]] +#shortname = "his" +#name = "History & Humanities" -[[archives]] -shortname = "int" -name = "International" +#[[archives]] +#shortname = "int" +#name = "International" -[[archives]] -shortname = "k" -name = "Weapons" +#[[archives]] +#shortname = "k" +#name = "Weapons" -[[archives]] -shortname = "m" -name = "Mecha" +#[[archives]] +#shortname = "m" +#name = "Mecha" -[[archives]] -shortname = "mlp" -name = "Pony" +#[[archives]] +#shortname = "mlp" +#name = "Pony" -[[archives]] -shortname = "q" -name = "4chan Feedback" +#[[archives]] +#shortname = "q" +#name = "4chan Feedback" -[[archives]] -shortname = "qa" -name = "Question & Answer" +#[[archives]] +#shortname = "qa" +#name = "Question & Answer" -[[archives]] -shortname = "r9k" -name = "ROBOT9001" +#[[archives]] +#shortname = "r9k" +#name = "ROBOT9001" -[[archives]] -shortname = "tg" -name = "Traditional Games" +#[[archives]] +#shortname = "tg" +#name = "Traditional Games" -[[archives]] -shortname = "trash" -name = "Off-Topic" +#[[archives]] +#shortname = "trash" +#name = "Off-Topic" -[[archives]] -shortname = "vr" -name = "Retro Games" +#[[archives]] +#shortname = "vr" +#name = "Retro Games" -[[archives]] -shortname = "wsg" -name = "Worksafe GIF" +#[[archives]] +#shortname = "wsg" +#name = "Worksafe GIF" [[archives]] shortname = "g" @@ -162,8 +162,8 @@ reports = false ## The following is the default for ayase. ## You may also specify a full URL. [image_location] -image = "http://127.0.0.1:7865/buckets/asagi-thumbs/{board_name}/image" -thumb = "http://127.0.0.1:7865/buckets/asagi-thumbs/{board_name}/thumb" +image = "http://127.0.0.1:42081/buckets/asagi-images/{board_name}/image" +thumb = "http://127.0.0.1:42081/buckets/asagi-thumbs/{board_name}/thumb" ## Torako sha256 folder structure. ## If you are using sha256 with seaweedfs, with thumbs and images in the same folder, diff --git a/docker-compose.yml b/docker-compose.yml index 72cc89f..16171d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: restart: always networks: - foolstack - ports: + #ports: # - 127:0.0.1:9333:9333 # - 127:0.0.1:19333:19333 volumes: @@ -23,7 +23,7 @@ services: - foolstack volumes: - ./seaweedfs/seaweed-volume-1:/data - ports: + #ports: # - 127:0.0.1:8080:8080 # - 127:0.0.1:18080:18080 command: 'volume -mserver="seaweed-master:9333" -port=8080 -dataCenter=asagi -rack=hdd -max=32' @@ -38,7 +38,7 @@ services: networks: - foolstack ports: - - 7865:7865 + - 42081:7865 ## Readonly port, for serving ayase images. # - 127:0.0.1:8888:8888 # - 127:0.0.1:18888:18888 environment: @@ -63,7 +63,7 @@ services: restart: always networks: - foolstack - ports: + #ports: # - 127:0.0.1:8333:8333 command: 's3 -filer="seaweed-filer:8888" -config=/config.json' volumes: @@ -91,26 +91,11 @@ services: MYSQL_DATABASE: 'asagi' networks: - foolstack - ports: + #ports: # - 127:0.0.1:3306:3306 volumes: - ./mysql/data:/var/lib/mysql - ayase: - image: suika/ayase:ayase - container_name: ayase - depends_on: - 4chan-scrape-db: - condition: service_started - restart: always - networks: - - foolstack - ports: - - 127:0.0.1:29080:8000 - command: ["uvicorn","view.asagi:app", "--host", "0.0.0.0"] - volumes: - - ./ayase.config.toml:/opt/ayase/config.toml - networks: foolstack: name: foolstack \ No newline at end of file diff --git a/mkdir.sh b/mkdir.sh deleted file mode 100644 index b448f38..0000000 --- a/mkdir.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -## mkdirs.sh -## Ensure required dirs exist. - -# cd /o/ - -## seaweed-master -mkdir -vp ./seaweedfs/master/data -chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/master/ ./seaweedfs/master/data - -## seaweed-volume-1 -mkdir -vp ./seaweedfs/seaweed-volume-1 -chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/seaweed-volume-1 - -## seaweed-filer -mkdir -vp ./seaweedfs/filer/filerldb2 -chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/filer ./seaweedfs/filer/filerldb2 diff --git a/startup.sh b/startup.sh new file mode 100644 index 0000000..3d13843 --- /dev/null +++ b/startup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +## startup.sh +## Startup foolstack in the proper order + +echo "startup.sh: Starting up foolstack SeaweedFS + Torako + Ayase..." +echo "startup.sh: This takes at least 1 minutes, go grab a coffee." + +## Startup seaweedfs + database containers +echo "startup.sh: First docker-compose (weed+mysql)..." +docker-compose -f "docker-compose.yml" up -d +echo "startup.sh: First docker-compose returned" + +sleep 30 ## Safety margin for containers to startup properly. + +## Ensure S3 buckets exist: +echo "startup.sh: Creating buckets..." +docker exec -i seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 < Date: Thu, 11 Feb 2021 23:33:55 +0800 Subject: [PATCH 4/4] Change back port number --- ayase.config.toml | 4 ++-- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ayase.config.toml b/ayase.config.toml index 1af5ca8..7abc077 100644 --- a/ayase.config.toml +++ b/ayase.config.toml @@ -162,8 +162,8 @@ reports = false ## The following is the default for ayase. ## You may also specify a full URL. [image_location] -image = "http://127.0.0.1:42081/buckets/asagi-images/{board_name}/image" -thumb = "http://127.0.0.1:42081/buckets/asagi-thumbs/{board_name}/thumb" +image = "http://127.0.0.1:7865/buckets/asagi-images/{board_name}/image" +thumb = "http://127.0.0.1:7865/buckets/asagi-thumbs/{board_name}/thumb" ## Torako sha256 folder structure. ## If you are using sha256 with seaweedfs, with thumbs and images in the same folder, diff --git a/docker-compose.yml b/docker-compose.yml index 16171d1..3124c8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: networks: - foolstack ports: - - 42081:7865 ## Readonly port, for serving ayase images. + - 127.0.0.1:7865:7865 ## Readonly port, for serving ayase images. # - 127:0.0.1:8888:8888 # - 127:0.0.1:18888:18888 environment: