diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9692323 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +## .gitignore + +## Private +git_ignored/ + +## Used by containers +seaweedfs/ +mysql/ + +## Clutter +.vscode/ +*.code-workspace +*.~* \ No newline at end of file diff --git a/README.md b/README.md index 65d451b..6677cf0 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ $ docker-compose -f "docker-compose.yml" up -d Create the buckets where the files will be located (this creates two different collections, basically own volumes different from others): ``` -docker exec seaweed-master -it /usr/bin/weed -master=seaweed-master:9333 -filer=seaweed-filer:8888 -s3.bucket.create -name asagi-thumbs -s3.bucket.create -name asagi-images +$ docker exec -it seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 +> s3.bucket.create -name asagi-thumbs +> s3.bucket.create -name asagi-images + ctrl+d/ctrl+c/exit ``` diff --git a/Torako.toml b/Torako.toml index 02c8439..9a7c437 100644 --- a/Torako.toml +++ b/Torako.toml @@ -10,7 +10,7 @@ api_addr = "127.0.0.1:2377" ## this rate limit only affects downloading the JSON threads/posts ## and will ensure api requests are throttled appropirately. ## If this option is not specified, there will be no rate limiting -#rate_limit = 10 +rate_limit = 2 ## Global board thread concurrency. This is a limit on the amount of ## of threads that will be retrieved and buffered at once. If this limit @@ -49,7 +49,7 @@ request_timeout = "30m" #host = "a.4cdn.org" ## Default refresh rate for boards (Default: 10s) -#refresh_rate = "10s" +refresh_rate = "30s" ## Default deleted page threshold. If threads ## are removed before this threshold is reached @@ -58,27 +58,27 @@ request_timeout = "30m" #deleted_page_threshold = 8 ## Download thumbnails (Default: true) -#download_thumbs = true +download_thumbs = true ## Download media/images (Default: true) -#download_media = true +download_media = true ## List the boards that should be archived -[boards.a] -[boards.c] -[boards.d] -[boards.e] -[boards.f] -[boards.fit] +#[boards.a] +#[boards.c] +#[boards.d] +#[boards.e] +#[boards.f] +#[boards.fit] [boards.g] -[boards.gif] -[boards.h] -[boards.ic] -[boards.jp] -[boards.u] -[boards.w] -[boards.wg] -[boards.vt] +#[boards.gif] +#[boards.h] +#[boards.ic] +#[boards.jp] +#[boards.u] +#[boards.w] +#[boards.wg] +#[boards.vt] ## Any of the above options can be provided ## here for board specific options @@ -112,7 +112,7 @@ old_dir_structure = false ## Sha256 Directory Structure. Images will be stored in accordance to the ## sha256 hash of the image. (Default: false) -sha_dir_structure = false +sha_dir_structure = true ## If we fail to save a thread or post to MySQL for whatever ## reason, we should consider this as a fatal error and @@ -149,7 +149,7 @@ sha_dir_structure = false ## concurrent downloads will ont exceed this number. ## Tune this number according to your memory and bandwidth preferences ## (Default: 128) -concurrent_downloads = 10 +concurrent_downloads = 16 ## Setting this to true means that the media download queue will backpressure ## the system, and once all concurrent_download slots are full, no more new @@ -289,7 +289,7 @@ compute_stats = true ## like MySQL, UTC mode must be set explictly. This can be safely disabled ## if your database is already in UTC. ## (Default: true) -#sql_set_utc = true +sql_set_utc = true [backend.asagi_pg_search] disabled = true diff --git a/ayase.config.toml b/ayase.config.toml index 844a49e..7abc077 100644 --- a/ayase.config.toml +++ b/ayase.config.toml @@ -24,81 +24,85 @@ authentication_type = "oauth2" ## List of archived boards to display. -[[archives]] -shortname = "a" -name = "Anime & Manga" +#[[archives]] +#shortname = "a" +#name = "Anime & Manga" -[[archives]] -shortname = "aco" -name = "Adult Cartoons" +#[[archives]] +#shortname = "aco" +#name = "Adult Cartoons" -[[archives]] -shortname = "an" -name = "Animals & Nature" +#[[archives]] +#shortname = "an" +#name = "Animals & Nature" -[[archives]] -shortname = "c" -name = "Cute" +#[[archives]] +#shortname = "c" +#name = "Cute" -[[archives]] -shortname = "co" -name = "Comics & Cartoons" +#[[archives]] +#shortname = "co" +#name = "Comics & Cartoons" -[[archives]] -shortname = "d" -name = "Hentai/Alternative" +#[[archives]] +#shortname = "d" +#name = "Hentai/Alternative" -[[archives]] -shortname = "fit" -name = "Fitness" +#[[archives]] +#shortname = "fit" +#name = "Fitness" -[[archives]] -shortname = "his" -name = "History & Humanities" +#[[archives]] +#shortname = "his" +#name = "History & Humanities" -[[archives]] -shortname = "int" -name = "International" +#[[archives]] +#shortname = "int" +#name = "International" -[[archives]] -shortname = "k" -name = "Weapons" +#[[archives]] +#shortname = "k" +#name = "Weapons" -[[archives]] -shortname = "m" -name = "Mecha" +#[[archives]] +#shortname = "m" +#name = "Mecha" -[[archives]] -shortname = "mlp" -name = "Pony" +#[[archives]] +#shortname = "mlp" +#name = "Pony" -[[archives]] -shortname = "q" -name = "4chan Feedback" +#[[archives]] +#shortname = "q" +#name = "4chan Feedback" -[[archives]] -shortname = "qa" -name = "Question & Answer" +#[[archives]] +#shortname = "qa" +#name = "Question & Answer" -[[archives]] -shortname = "r9k" -name = "ROBOT9001" +#[[archives]] +#shortname = "r9k" +#name = "ROBOT9001" -[[archives]] -shortname = "tg" -name = "Traditional Games" +#[[archives]] +#shortname = "tg" +#name = "Traditional Games" -[[archives]] -shortname = "trash" -name = "Off-Topic" +#[[archives]] +#shortname = "trash" +#name = "Off-Topic" -[[archives]] -shortname = "vr" -name = "Retro Games" +#[[archives]] +#shortname = "vr" +#name = "Retro Games" + +#[[archives]] +#shortname = "wsg" +#name = "Worksafe GIF" [[archives]] -shortname = "wsg" -name = "Worksafe GIF" +shortname = "g" +name = "Computer nerd shit" ## List of non-archive boards to display. ## Boards currently are no different from archives, although it may be used for text diff --git a/docker-compose.yml b/docker-compose.yml index d97a763..3124c8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: restart: always networks: - foolstack - ports: + #ports: # - 127:0.0.1:9333:9333 # - 127:0.0.1:19333:19333 volumes: @@ -23,10 +23,10 @@ services: - foolstack volumes: - ./seaweedfs/seaweed-volume-1:/data - ports: + #ports: # - 127:0.0.1:8080:8080 # - 127:0.0.1:18080:18080 - command: 'volume -mserver="seaweed-master:9333" -port=8080 -dataCenter=asagi -rack=hdd' + command: 'volume -mserver="seaweed-master:9333" -port=8080 -dataCenter=asagi -rack=hdd -max=32' depends_on: seaweed-master: condition: service_started @@ -38,14 +38,19 @@ services: networks: - foolstack ports: - - 7865:7865 + - 127.0.0.1:7865:7865 ## Readonly port, for serving ayase images. # - 127:0.0.1:8888:8888 # - 127:0.0.1:18888:18888 - command: 'filer -master="seaweed-master:9333" -disableDirListing -port.readonly=7865' + environment: + WEED_LEVELDB2_ENABLED: 'true' + WEED_LEVELDB2_DIR: '/filerldb2' + #command: 'filer -master="seaweed-master:9333" -disableDirListing -port.readonly=7865' + command: 'filer -master="seaweed-master:9333" -port.readonly=7865' tty: true stdin_open: true volumes: - ./seaweedfs/filer:/data + - ./seaweedfs/filer/filerldb2:/filerldb2 depends_on: seaweed-master: condition: service_started @@ -58,7 +63,7 @@ services: restart: always networks: - foolstack - ports: + #ports: # - 127:0.0.1:8333:8333 command: 's3 -filer="seaweed-filer:8888" -config=/config.json' volumes: @@ -86,7 +91,7 @@ services: MYSQL_DATABASE: 'asagi' networks: - foolstack - ports: + #ports: # - 127:0.0.1:3306:3306 volumes: - ./mysql/data:/var/lib/mysql diff --git a/misc_notes.md b/misc_notes.md new file mode 100644 index 0000000..4db7bce --- /dev/null +++ b/misc_notes.md @@ -0,0 +1,12 @@ + +# 2020-02-11 suika latest version in branch "ayase" + + +## Create buckets by hand: +$ docker exec -it seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 +> s3.bucket.create -name asagi-thumbs +> s3.bucket.create -name asagi-images + + + + diff --git a/mkdir.sh b/mkdirs.sh old mode 100644 new mode 100755 similarity index 99% rename from mkdir.sh rename to mkdirs.sh index b448f38..77ce413 --- a/mkdir.sh +++ b/mkdirs.sh @@ -15,3 +15,4 @@ chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/seaweed-volume-1 ## seaweed-filer mkdir -vp ./seaweedfs/filer/filerldb2 chmod -v "u=rwx,g=rwx,o=rwx" ./seaweedfs/filer ./seaweedfs/filer/filerldb2 + diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..9fbc138 --- /dev/null +++ b/readme.md @@ -0,0 +1,99 @@ +# Foolfstack - torako + seaweedfs + ayase +Quick guide to using this stuff. +Non-local access is outside the current scope. + +## Install preprequisites +* Install docker + +https://docs.docker.com/engine/install/fedora/ +``` +## Update system packages +$ sudo dnf -y update +$ sudo dnf -y install dnf-plugins-core + +## Add docker repository: +$ sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo +$ sudo dnf makecache + +## Install docker packages: +$ sudo dnf install docker-ce docker-ce-cli containerd.io + +## Start docker service: +$ sudo systemctl enable --now docker +$ systemctl status docker + +## Add current user to docker group: +$ sudo usermod -aG docker $(whoami) + +## "login" to docker group in an existing terminal session: +$ newgrp docker +``` + +* Install docker-compose +``` +$ sudo dnf install docker-compose +``` + + +## Prep +Seaweedfs does not implicaitly create many of its dirs. +This script makes them if they do not exist and assigns permissions permissive enough to make sure seaweedfs can use them. +``` +$ bash mkdirs.sh +``` + + +## Running: +Start/create containers in the background: +``` +$ docker-compose -f "docker-compose.yml" up -d +``` + +Stop/remove containers: +``` +$ docker-compose -f "docker-compose.yml" down +``` + + +### Viewing logs +``` +$ docker-compose -f "docker-compose.yml" logs +$ docker-compose -f "docker-compose.yml" logs ayase +``` + +### Running commands inside containers manually: +``` +$ docker exec -it 4chan-scrape-db sh +container$ mysql -uroot -pbunbuncha +container.mysql> SELECT host, user FROM mysql.user; +``` + +Installing packages in container for diagnostics: +``` +apt-get update && apt-get install -y traceroute curl iputils +``` + + +## Ayase website +In a web browser, open this address to view the ayase instance provided by the container "ayase": + +`http://127:0.0.1:29080/` + + +## DB Export +* Note: The method demonstrated does not strip secrets such as deletion passwords or IP addresses from the generated file. + +Backup the entire database: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables --all-databases | gzip > "4chan-scrape-db.all-databases.t`date -u +%s`.sql.gz" +``` + +Export a single board: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables "asagi" "g" "g_images" | gzip > "4chan-scrape-db.g.t`date -u +%s`.sql.gz" +``` + +Export multiple boards: +``` +$ docker exec "4chan-scrape-db" /usr/bin/mysqldump -h"localhost" -u"root" -p"bunbuncha" --tz-utc --quick --opt --single-transaction --skip-lock-tables "asagi" "g" "g_images" "co" "co_images" | gzip > "4chan-scrape-db.my_boards_export.t`date -u +%s`.sql.gz" +``` diff --git a/startup.sh b/startup.sh new file mode 100644 index 0000000..3d13843 --- /dev/null +++ b/startup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +## startup.sh +## Startup foolstack in the proper order + +echo "startup.sh: Starting up foolstack SeaweedFS + Torako + Ayase..." +echo "startup.sh: This takes at least 1 minutes, go grab a coffee." + +## Startup seaweedfs + database containers +echo "startup.sh: First docker-compose (weed+mysql)..." +docker-compose -f "docker-compose.yml" up -d +echo "startup.sh: First docker-compose returned" + +sleep 30 ## Safety margin for containers to startup properly. + +## Ensure S3 buckets exist: +echo "startup.sh: Creating buckets..." +docker exec -i seaweed-master /usr/bin/weed shell -master=seaweed-master:9333 -filer=seaweed-filer:8888 <