diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f84046 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +tmp +.env diff --git a/.tool-versions b/.tool-versions index 009ed9c..8279d55 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ uv 0.6.1 -golang 1.22.12 +golang 1.23.2 +tilt 0.33.21 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 41b3f37..648e355 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,9 +11,27 @@ Add yourself to http://62.84.113.17/ to get notified about changes. This describes how to run the repo locally. -## Prerequisites +### Prerequisites This repository is using [.tool-versions](https://asdf-vm.com/manage/configuration.html). [Mise](https://github.com/jdx/mise) is recommended for managing the versions of the tools. Integrate it with your shell and you should have all the needed tools installed. By cd in the root of the repository, mise will download and install the needed versions of the tools, except for docker-compose and docker. +Why docker-compose and docker are not included in .tool-versions? They are heavy for the system, many options to have +them exists (podman, orbstack, colima, docker, etc, etc). You need to configure them yourself. + +This uses: + +* [tilt-dev](https://tilt.dev/) +* docker-compose + +### Run the local setup + +The following command must be enough to run minimal setup: + +```bash +tilt up +``` + +If you see warnings in the console about missing environment keys, +that means your setup is not complete. You need to provide the missing keys in `.env` file. diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 0000000..afc50e0 --- /dev/null +++ b/Tiltfile @@ -0,0 +1,98 @@ +# -*- mode: Python -*- + +# +# Command line args +# + +# Specifies a list of args like `tilt up foo bar blabla` +config.define_string_list("to-run", args=True) +cfg = config.parse() + + +TILT_MODE_DEFAULT = "default" + +# minimal subset of resources to develop +default_resources = [ + "telegramsearch", + "telegramsearch-pg", + +] + +groups = { + TILT_MODE_DEFAULT: default_resources, +} + +# Resources to run, empty array means everything +resources = [] + +# Run TILT_MODE_DEFAULT by default +tilt_args = cfg.get("to-run", [TILT_MODE_DEFAULT]) + +for arg in tilt_args: + if arg in groups: + resources += groups[arg] + else: + # Also support specifying individual services instead of groups, + # e.g. `tilt up a b d` + resources.append(arg) + + +# Tells Tilt to only run specified resources +config.set_enabled_resources(resources) + +# +# Plugins +# + +load("ext://uibutton", "cmd_button") + +# +# Docker compose +# + +docker_compose_files = [ + "./docker-compose.yml", + "./docker-compose.persistent.yml", +] +docker_compose(docker_compose_files) + +# +# Labels +# + +dc_resource( + "telegramsearch", + labels=["bot"], + resource_deps=[ + "telegramsearch-pg", + ], +) + +dc_resource( + "telegramsearch-pg", + labels=["bot"], +) + +local_resource( + "migrate telegramsearch-pg", + cmd="DOCKER_BUILDKIT=1 docker build . -f app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile -t yanakipre/telegramsearch-migrations:local && docker run --network yanakipre_net --env DATABASE_URL='postgres://postgres:password@10.30.41.52:5432/telegramsearch' yanakipre/telegramsearch-migrations:local", + labels=["bot"], + auto_init=False, + trigger_mode=TRIGGER_MODE_MANUAL, + resource_deps=[ + "telegramsearch-pg", + ], +) + +docker_build( + "yanakipre/telegramsearch:local", + context=".", + dockerfile="app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile", + only=[ + "app/telegramsearch", + "internal", + "go.mod", + "go.sum", + ], +) + diff --git a/app/archwg-router/.gitignore b/app/archwg-router/.gitignore new file mode 100644 index 0000000..1d17dae --- /dev/null +++ b/app/archwg-router/.gitignore @@ -0,0 +1 @@ +.venv diff --git a/app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile b/app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile new file mode 100644 index 0000000..67cff5f --- /dev/null +++ b/app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile @@ -0,0 +1,86 @@ +# Use BuildKit’s Dockerfile frontend to cache local builds. +# https://www.docker.com/blog/containerize-your-go-developer-environment-part-2/ +# +# Run docker with DOCKER_BUILDKIT=1 prepended to docker command to activate Build Kit backend. +# +# On linux +# cat /etc/docker/daemon.json +#{ "features": { "buildkit": true } } +# syntax = docker/dockerfile:1-experimental + + +# +# Telegram bot image +# + +# +# Intermediate image to build golang API server +# + +# Image version version is sha256 to speed up builds (especially local ones) and keep builds reproducible. +# 1.23.2-bullseye +FROM golang@sha256:ecb3fe70e1fd6cef4c5c74246a7525c3b7d59c48ea0589bbb0e57b1b37321fb9 AS go-build +ARG DEBUG=false + +# define parent caching directory +ENV CACHE=/cache + +# define go caching directories +ENV GOCACHE=$CACHE/gocache +ENV GOMODCACHE=$CACHE/gomodcache + +WORKDIR /build + +# cache dependencies before building +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,source=go.mod,target=go.mod \ + --mount=type=bind,source=go.sum,target=go.sum \ + go mod download -x + + +RUN mkdir /debugbin; \ + if [ "$DEBUG" = "true" ]; then \ + GOBIN=/debugbin/ go install github.com/go-delve/delve/cmd/dlv@v1.23.1; \ + fi + +RUN apt-get update && apt-get install -y git + +# build executable, store in cache so rebuilds can no-op +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,target=. \ + if [ "$DEBUG" = "true" ]; then \ + go build -buildvcs=auto -o $CACHE/telegramsearch -gcflags="all=-N -l" ./app/telegramsearch/cmd/telegramsearch && cp $CACHE/telegramsearch /bin; \ + else \ + go build -buildvcs=auto -o $CACHE/telegramsearch ./app/telegramsearch/cmd/telegramsearch && cp $CACHE/telegramsearch /bin; \ + fi +# IIUC, there is no such thing as release build in Go. It bundles +# debug symbols by default. We can strip them with `go build -ldflags "-s -w"`, +# but why may want that? + +# +# Final image to be exported +# +# Image version is fixed to speed up builds (especially local ones) and keep builds reproducible. +# bullseye-slim +FROM debian@sha256:60a596681410bd31a48e5975806a24cd78328f3fd6b9ee5bc64dca6d46a51f29 + +# it's an antipattern to run the next lines in multiple steps, +# but we're having frequent 4294967295 exit code and this is an attempt to debug it. +# after it's solved - combine the steps into single one. +RUN apt-get update || cat /var/log/apt/*.log +RUN apt-get install --yes --no-install-recommends \ + ca-certificates \ + openssl || cat /var/log/apt/*.log +RUN apt-get clean || cat /var/log/apt/*.log + +RUN useradd -d /example yanakipre +USER yanakipre +WORKDIR /yanakipre + +# Only copy if the /debugbin/dlv exists, otherwise it will do nothing +COPY --chown=yanakipre --from=go-build /debugbin/dlv* /usr/local/bin/ +COPY --from=go-build /bin/telegramsearch /yanakipre/telegramsearch + +EXPOSE 3000 9090 +ENTRYPOINT ["/yanakipre/telegramsearch"] +CMD [""] diff --git a/app/telegramsearch/cmd/telegramsearch/telegramsearch.yaml.example b/app/telegramsearch/cmd/telegramsearch/telegramsearch.yaml.example new file mode 100644 index 0000000..cb285ab --- /dev/null +++ b/app/telegramsearch/cmd/telegramsearch/telegramsearch.yaml.example @@ -0,0 +1,68 @@ +ctlv1: + help_text: | + Добрый день! Это - бот, который поможет вам найти ответы на ваши вопросы. + + Чтобы задать вопрос, просто напишите его в чат. Он постарается найти наиболее подходящий ответ на ваш вопрос. + Бот не выдумывает от себя, все его данные собраны от живых людей. + + Я, разработчик, буду очень признателен, если вы поделитесь своими впечатлениями о боте и порекомендуете его своим друзьям, если он вам полезен. + + Вот тут можно связаться с разработчиком: https://substack.com/home/post/p-148053843 + news_text: | + Новости: + + Мы открыли чат, в котором все вопросы можно задавать и боту, и людям. Присоединяйтесь: https://t.me/+8trW_-0GEFI1NTE0 + Мы планируем добавить в бота функцию показа ссылок на чаты, где было найдена эта информация. + Это позволит вам быстро найти чаты, где обсуждаются интересные вам темы. + Если вам есть что прокомментировать или добавить - пишите в чате https://t.me/+8trW_-0GEFI1NTE0 или в https://substack.com/home/post/p-148053843 + no_explained_answer: К сожалению, у меня нет информации об этом сообщении. Возможно + оно было задано слишком давно и я о нем забыл. + noresultsanswer: К сожалению, у меня недостаточно информации чтобы ответить на данный + вопрос. Но я учусь и, вероятно, смогу ответить на него позже. + staleresponsestext: В ответе не использовано информации свежее чем от %s + freshresponsestext: 'Обсуждений: %d' + stalethreshold: 17520h0m0s +postgres_rw: + rdb: + search_path: "" + database_type: "" + database_url: xxx + max_conn_lifetime: 1h0m0s + max_open_conns: 50 + collect_metrics_interval: 5s +openai: + embeddingconfig: + model: text-embedding-3-small + apikey: xxx + transport: + client_name: openapi + timeout: 0s + dial_timeout: 1m0s + dial_keep_alive: 5m0s + idle_conn_timeout: 3m0s + tls_handshake_timeout: 10s + expect_continue_timeout: 1s + response_header_timeout: 1m0s + disable_keep_alives: false + max_idle_conns: 100 + max_idle_conns_per_host: 2 + max_conns_per_host: 0 + retries: + backoff: 100ms + attempts: 10 + rate_limiters: [] + asking_about: Cyprus + donothighlight: Cyprus +logging: + sink: stdout + log_level: DEBUG + log_format: json + filters: + by_logger_name: [] + by_exact_name: [] +telegram_transport: + token: xxx + greeting: "" +telegram_v2: + appid: xxx + apphash: xxx diff --git a/app/telegramsearch/internal/pkg/staticconfig/loader.go b/app/telegramsearch/internal/pkg/staticconfig/loader.go index 482d995..fbb2418 100644 --- a/app/telegramsearch/internal/pkg/staticconfig/loader.go +++ b/app/telegramsearch/internal/pkg/staticconfig/loader.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/heetch/confita" "github.com/heetch/confita/backend" ) @@ -12,12 +11,16 @@ import ( // Backend specifies custom logic for config loading type Backend struct{} -func (b Backend) Unmarshal(ctx context.Context, to any) error { - _, ok := to.(*Config) +func (b Backend) Unmarshal(_ context.Context, to any) error { + cfg, ok := to.(*Config) if !ok { return fmt.Errorf("cannot unmarshall to Config: %+v", to) } + cfg.PostgresRW.RDB.DSN.FromEnv("DATABASE_URL") + cfg.OpenAI.ApiKey.FromEnv("OPENAI_API_KEY") + cfg.TelegramTransport.Token.FromEnv("TELEGRAM_BOT_TOKEN") + return nil } diff --git a/app/telegramsearch/telegramsearch-db/generate.go b/app/telegramsearch/telegramsearch-db/generate.go index 4df538b..4c4dd73 100644 --- a/app/telegramsearch/telegramsearch-db/generate.go +++ b/app/telegramsearch/telegramsearch-db/generate.go @@ -1,3 +1,3 @@ package telegramsearchdb -//go:generate go run ../db-management/cmd/db-freeze --dbpath=. +//go:generate go run ../../../internal/db-management/cmd/db-freeze --dbpath=. diff --git a/app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile b/app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile new file mode 100644 index 0000000..dbed316 --- /dev/null +++ b/app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile @@ -0,0 +1,41 @@ +# Image version version is sha256 to speed up builds (especially local ones) and keep builds reproducible. +# 1.23.2-alpine +FROM golang@sha256:9dd2625a1ff2859b8d8b01d8f7822c0f528942fe56cfe7a1e7c38d3b8d72d679 AS build + + +WORKDIR /build + +COPY go.mod . +COPY go.sum . + +# define parent caching directory +ENV CACHE=/cache + +# define go caching directories +ENV GOCACHE=$CACHE/gocache +ENV GOMODCACHE=$CACHE/gomodcache + +# cache dependencies before building +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,source=go.mod,target=go.mod \ + --mount=type=bind,source=go.sum,target=go.sum \ + go mod download -x + +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,target=. \ + go build -o $CACHE/postgres-migrate ./internal/db-management/cmd/postgres-migrate && cp $CACHE/postgres-migrate /bin/ + +# Image version is fixed to speed up builds (especially local ones) and keep builds reproducible. +# 3.14.10 +FROM alpine@sha256:0f2d5c38dd7a4f4f733e688e3a6733cb5ab1ac6e3cb4603a5dd564e5bfb80eed + +RUN apk add --no-cache bash ca-certificates + +# https://github.com/moby/moby/issues/30081#issuecomment-1137512170 +RUN mkdir -p /db +WORKDIR /db + +COPY --from=build /bin/postgres-migrate /usr/local/bin/postgres-migrate +COPY app/telegramsearch/telegramsearch-app/migrations /db/migrations + +CMD ["postgres-migrate"] diff --git a/app/telegramsearch/telegramsearch.Dockerfile b/app/telegramsearch/telegramsearch.Dockerfile new file mode 100644 index 0000000..264e777 --- /dev/null +++ b/app/telegramsearch/telegramsearch.Dockerfile @@ -0,0 +1,86 @@ +# Use BuildKit’s Dockerfile frontend to cache local builds. +# https://www.docker.com/blog/containerize-your-go-developer-environment-part-2/ +# +# Run docker with DOCKER_BUILDKIT=1 prepended to docker command to activate Build Kit backend. +# +# On linux +# cat /etc/docker/daemon.json +#{ "features": { "buildkit": true } } +# syntax = docker/dockerfile:1-experimental + + +# +# Telegramsearch app Docker image +# + +# +# Intermediate image to build golang API server +# + +# Image version version is sha256 to speed up builds (especially local ones) and keep builds reproducible. +# 1.23.2-bullseye +FROM golang@sha256:ecb3fe70e1fd6cef4c5c74246a7525c3b7d59c48ea0589bbb0e57b1b37321fb9 AS go-build +ARG DEBUG=false + +# define parent caching directory +ENV CACHE=/cache + +# define go caching directories +ENV GOCACHE=$CACHE/gocache +ENV GOMODCACHE=$CACHE/gomodcache + +WORKDIR /build + +# cache dependencies before building +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,source=go.mod,target=go.mod \ + --mount=type=bind,source=go.sum,target=go.sum \ + go mod download -x + +RUN mkdir /debugbin; \ + if [ "$DEBUG" = "true" ]; then \ + GOBIN=/debugbin/ go install github.com/go-delve/delve/cmd/dlv@v1.23.1; \ + fi + +RUN apt-get update && apt-get install -y git + +# build executable, store in cache so rebuilds can no-op +RUN --mount=type=cache,target=$CACHE \ + --mount=type=bind,target=. \ + if [ "$DEBUG" = "true" ]; then \ + go build -buildvcs=auto -o $CACHE/api -gcflags="all=-N -l" ./app/example-app/cmd/api && cp $CACHE/api /bin; \ + else \ + go build -buildvcs=auto -o $CACHE/api ./app/example-app/cmd/api && cp $CACHE/api /bin; \ + fi + +# IIUC, there is no such thing as release build in Go. It bundles +# debug symbols by default. We can strip them with `go build -ldflags "-s -w"`, +# but why may want that? + +# +# Final image to be exported +# +# Image version is fixed to speed up builds (especially local ones) and keep builds reproducible. +# bullseye-slim +FROM debian@sha256:60a596681410bd31a48e5975806a24cd78328f3fd6b9ee5bc64dca6d46a51f29 + +# it's an antipattern to run the next lines in multiple steps, +# but we're having frequent 4294967295 exit code and this is an attempt to debug it. +# after it's solved - combine the steps into single one. +RUN apt-get update || cat /var/log/apt/*.log +RUN apt-get install --yes --no-install-recommends \ + ca-certificates \ + openssl || cat /var/log/apt/*.log +RUN apt-get clean || cat /var/log/apt/*.log + +RUN useradd -d /example everypay +USER everypay +WORKDIR /example + +# Only copy if the /debugbin/dlv exists, otherwise it will do nothing +COPY --chown=everypay --from=go-build /debugbin/dlv* /usr/local/bin/ +COPY --from=go-build /bin/api /example/api + +EXPOSE 3000 9090 +ENTRYPOINT ["/example/api"] +CMD [""] diff --git a/docker-compose.persistent.yml b/docker-compose.persistent.yml new file mode 100644 index 0000000..896c552 --- /dev/null +++ b/docker-compose.persistent.yml @@ -0,0 +1,8 @@ +# Extracted volume settings. +# See https://docs.docker.com/compose/extends/#adding-and-overriding-configuration + +version: "3.9" +services: + telegramsearch-pg: + volumes: + - "${RUNNER_DIR:-.}/tmp/example-pg:/var/lib/postgresql/data" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..726b9a9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,62 @@ +# Adding new services here without explicitly defining the ipv4_address +# breaks the k8s cluster assumption, that k3d is running on 10.30.42.2. +# So, specify an address explicitly, do not use 10.30.42.2. +version: "3.9" + +services: + telegramsearch: + image: yanakipre/telegramsearch:local + build: + context: . + dockerfile: app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile + command: + - "telegram" + - "bot" + ports: [ ] + hostname: telegramsearch.yanakipre.local + environment: + LOG_FORMAT: console + OPENAI_API_KEY: ${OPENAI_API_KEY} + DATABASE_URL: "postgres://postgres:password@telegramsearch-pg:5432/bot" + + extra_hosts: + - "host.docker.internal:host-gateway" + - "telegramsearch-pg:10.30.41.52" + networks: + yanakipre_net: + ipv4_address: 10.30.41.53 + depends_on: + - telegramsearch-pg + secrets: [ ] + + telegramsearch-pg: + # ankane/pgvector:v0.5.1 + image: ankane/pgvector@sha256:d3a9d8ac27bb7e05e333ef25b634d2625adaa85336ab729954b9e94859bf6fa7 + ports: + - "11432:5432" + environment: + POSTGRES_DB: bot + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + yanakipre_net: + ipv4_address: 10.30.41.52 + +# We use pre-created subnet to make it +# more reliable. +networks: + yanakipre_net: + name: yanakipre_net + ipam: + config: + - subnet: "10.30.41.0/24" + gateway: 10.30.41.1 + +volumes: + reserved: + +secrets: { } + +configs: { } diff --git a/go.mod b/go.mod index 50ff7e9..5302e6e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/yanakipre/bot -go 1.22.12 +go 1.23.2 replace ( k8s.io/api => k8s.io/api v0.26.15 @@ -79,7 +79,6 @@ require ( github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/gorilla/securecookie v1.1.2 // indirect github.com/gotd/ige v0.2.2 // indirect github.com/gotd/neo v0.1.5 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -135,9 +134,7 @@ require ( go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/mod v0.19.0 // indirect golang.org/x/sys v0.22.0 // indirect @@ -164,9 +161,7 @@ require ( github.com/go-faster/jx v1.1.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/gorilla/sessions v1.2.1 github.com/gotd/td v0.108.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/heetch/confita v0.10.0 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pgx/v4 v4.18.3 @@ -184,7 +179,6 @@ require ( github.com/redis/go-redis/v9 v9.0.5 github.com/rekby/fixenv v0.7.0 github.com/reugn/go-quartz v0.13.0 - github.com/rs/cors v1.8.2 github.com/samber/lo v1.47.0 github.com/samber/slog-zap/v2 v2.6.2 github.com/sashabaranov/go-openai v1.26.1 @@ -203,6 +197,7 @@ require ( go.opentelemetry.io/otel/sdk v1.25.0 go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.25.0 golang.org/x/net v0.27.0 golang.org/x/sync v0.8.0 golang.org/x/time v0.5.0 diff --git a/go.sum b/go.sum index 96d0c8b..30af281 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,6 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -307,11 +305,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= -github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= -github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= -github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gotd/ige v0.2.2 h1:XQ9dJZwBfDnOGSTxKXBGP4gMud3Qku2ekScRjDWWfEk= github.com/gotd/ige v0.2.2/go.mod h1:tuCRb+Y5Y3eNTo3ypIfNpQ4MFjrnONiL2jN2AKZXmb0= @@ -348,8 +341,6 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -616,8 +607,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -758,8 +747,6 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= diff --git a/internal/db-management/cmd/db-freeze/.gitignore b/internal/db-management/cmd/db-freeze/.gitignore new file mode 100644 index 0000000..d45810e --- /dev/null +++ b/internal/db-management/cmd/db-freeze/.gitignore @@ -0,0 +1 @@ +db-freeze diff --git a/internal/db-management/cmd/db-freeze/README.md b/internal/db-management/cmd/db-freeze/README.md new file mode 100644 index 0000000..9f63c0a --- /dev/null +++ b/internal/db-management/cmd/db-freeze/README.md @@ -0,0 +1,3 @@ +# db-freeze + +This utility applies migrations to a running Postgres database and dumps the result into a file using pgdump. diff --git a/internal/db-management/cmd/db-freeze/db-freeze.go b/internal/db-management/cmd/db-freeze/db-freeze.go new file mode 100644 index 0000000..14708d1 --- /dev/null +++ b/internal/db-management/cmd/db-freeze/db-freeze.go @@ -0,0 +1,235 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + _ "github.com/jackc/pgx/v5/stdlib" // postgres driver + "github.com/orlangure/gnomock" + "golang.org/x/crypto/sha3" + + "github.com/yanakipre/bot/internal/logger" + "github.com/yanakipre/bot/internal/pgtooling" + "github.com/yanakipre/bot/internal/rdb" + "github.com/yanakipre/bot/internal/rdb/rdbtesttooling" + "github.com/yanakipre/bot/internal/secret" + "github.com/yanakipre/bot/internal/testtooling" +) + +func main() { + logger.SetNewGlobalLoggerOnce(logger.DefaultConfig()) + + // This does not work ATM. + container, err := rdbtesttooling.StartPGContainer("ankane/pgvector@sha256:d3a9d8ac27bb7e05e333ef25b634d2625adaa85336ab729954b9e94859bf6fa7") + if err != nil { + panic(fmt.Sprintf("could not start postgres: %v", err)) + } + defer func() { _ = gnomock.Stop(container) }() + dsn := secret.NewString(fmt.Sprintf( + "postgres://postgres:password@%s:%d/postgres", + container.Host, + container.DefaultPort(), + )) + ctx := context.Background() + cfg := rdb.DefaultConfig() + cfg.DSN = dsn + + for i, dbPath := range pathsToDBs { + dbName := dbNames[i] + + logger.Debug(ctx, fmt.Sprintf("migrating db: %s from path: %s", dbName, dbPath)) + + teardown, err := migrateSingleDB(ctx, cfg, dbPath, dbName) + if teardown != nil { + defer func() { _ = teardown() }() + } + if err != nil { + panic(err) + } + } +} + +func migrateSingleDB(ctx context.Context, cfg rdb.Config, dbPath, dbName string) (func() error, error) { + db, teardown, err := testtooling.SetupDBWithName(ctx, cfg, dbName) + if err != nil { + return teardown, fmt.Errorf("could not set up db: %w", err) + } + + connCfg, err := pgx.ParseConfig(db.DSN().Unmask()) + if err != nil { + return teardown, fmt.Errorf("could not parse config: %w", err) + } + + opts := pgtooling.MigrateOpts{ + Destination: "last", + PathToDBDir: dbPath, + SchemaVersionTable: tableName, + } + opts.SetDefaults() + + err = pgtooling.Migrate( + ctx, + connCfg, + opts, + ) + if err != nil { + return teardown, fmt.Errorf("could not migrate: %w", err) + } + currentDbState, err := pgtooling.FormattedPgDump(db.DSN()) + if err != nil { + return teardown, fmt.Errorf("could not get pg dump: %w", err) + } + err = os.WriteFile( //nolint:gosec + filepath.Join(dbPath, "db.sql"), + []byte(currentDbState), + 0o644, + ) //nolint:gosec + if err != nil { + return teardown, fmt.Errorf("could not write db.sql: %w", err) + } + + err = dumpMigrationVersion(dbPath) + if err != nil { + return teardown, fmt.Errorf("could not write last_migration.version: %w", err) + } + + return teardown, nil +} + +func dumpMigrationVersion(dbPath string) error { + // We use git to list the files in the migrations directory + // to avoid including temporary files, work-in-progress migrations, etc. + cmd := exec.Command("git", "ls-files", dbPath+"migrations/*.sql") + output, err := cmd.Output() + if err != nil { + return err + } + // Handle both Windows CRLF and Unix LF line endings + cleaned := strings.ReplaceAll(string(output), "\r\n", "\n") + fs := strings.Split(strings.TrimSpace(cleaned), "\n") + + type Migration struct { + name string + version int + } + migrations := make([]Migration, 0, len(fs)) + for _, fileName := range fs { + f := filepath.Base(fileName) + if f == "events" { + continue + } + version, err := strconv.Atoi(strings.Split(f, "_")[0]) + if err != nil { + return err + } + migrations = append(migrations, Migration{name: f, version: version}) + } + sort.Slice( + migrations, + func(i, j int) bool { return migrations[i].version < migrations[j].version }, + ) + + lastMigration := migrations[len(migrations)-1] + + // take hash of file + name to avoid collisions + bytes, err := os.ReadFile(dbPath + "migrations/" + lastMigration.name) //nolint:gosec + if err != nil { + return err + } + bytes = append(bytes, []byte(lastMigration.name)...) + + lastMigrationBytes, err := json.Marshal(struct { + Version int `json:"version"` + Hash string `json:"hash"` + }{ + Version: lastMigration.version, + Hash: fmt.Sprintf("%X", sha3.Sum256(bytes)), + }) + if err != nil { + return err + } + + err = os.WriteFile( //nolint:gosec + filepath.Join(dbPath, "last_migration.json"), + lastMigrationBytes, + 0o644, + ) //nolint:gosec + if err != nil { + return err + } + + return nil +} + +var ( + pathToDB string + + pathsToDBsStr string + dbNamesStr string + + pathsToDBs []string + dbNames []string + + tableName string +) + +func init() { + flag.StringVar(&pathToDB, "dbpath", "", "path to database files") + flag.StringVar(&pathsToDBsStr, "dbpaths", "", "paths to database files") + flag.StringVar(&dbNamesStr, "dbnames", "", "names of dbs to migrate") + flag.StringVar(&tableName, "table", "", "table name with migration version") + flag.Parse() + + if err := validateArgs(); err != nil { + panic(err) + } + + if err := parseDBPathsAndNames(); err != nil { + panic(err) + } +} + +func validateArgs() error { + if pathToDB != "" && pathsToDBsStr != "" { + return errors.New("dbpath and dbpaths must not both be set") + } + if pathToDB == "" { + if pathsToDBsStr == "" { + return errors.New("either dbpath or dbpaths must be set") + } + if dbNamesStr == "" { + return errors.New("dbnames must be set with dbpaths and be of equal length") + } + } + return nil +} + +func parseDBPathsAndNames() error { + if pathsToDBsStr == "" { + pathsToDBs = []string{pathToDB} + dbNames = []string{"dbfreeze"} + } else { + pathsToDBs = strings.Split(pathsToDBsStr, ",") + dbNames = strings.Split(dbNamesStr, ",") + } + + if len(pathsToDBs) != len(dbNames) { + return errors.New("dbnames must be set with dbpaths and be of equal length") + } + + for i := range pathsToDBs { + pathsToDBs[i] = strings.TrimRight(pathsToDBs[i], "/") + "/" + } + + return nil +} diff --git a/internal/db-management/cmd/postgres-migrate/.gitignore b/internal/db-management/cmd/postgres-migrate/.gitignore new file mode 100644 index 0000000..3ab4601 --- /dev/null +++ b/internal/db-management/cmd/postgres-migrate/.gitignore @@ -0,0 +1 @@ +postgres-migrate diff --git a/internal/db-management/cmd/postgres-migrate/README.md b/internal/db-management/cmd/postgres-migrate/README.md new file mode 100644 index 0000000..17f802f --- /dev/null +++ b/internal/db-management/cmd/postgres-migrate/README.md @@ -0,0 +1,3 @@ +# Postgres Migrate + +This binary migrates postgres databases. diff --git a/internal/db-management/cmd/postgres-migrate/postgresmigrate.go b/internal/db-management/cmd/postgres-migrate/postgresmigrate.go new file mode 100644 index 0000000..f3213aa --- /dev/null +++ b/internal/db-management/cmd/postgres-migrate/postgresmigrate.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "errors" + "flag" + "os" + "os/signal" + "syscall" + + "github.com/jackc/pgx/v5" + "github.com/yanakipre/bot/internal/logger" + "github.com/yanakipre/bot/internal/pgtooling" + "go.uber.org/zap" +) + +func main() { + flag.Parse() + ctx, defaultBehaviourForSignals := signal.NotifyContext( + context.Background(), + syscall.SIGINT, + syscall.SIGTERM, + ) + defer defaultBehaviourForSignals() + logger.SetNewGlobalLoggerOnce(logger.DefaultConfig()) + lg := logger.FromContext(ctx) + defer lg.Info("application shutting down") + + config, err := pgx.ParseConfig(os.Getenv("DATABASE_URL")) + if err != nil { + logger.Fatal(ctx, "cannot create pgx config", zap.Error(err)) + } + + opts := pgtooling.MigrateOpts{ + Destination: flagDestination, + PathToDBDir: ".", + SchemaVersionTable: tableName, + } + + opts.SetDefaults() + + if err := pgtooling.Migrate(ctx, config, opts); err != nil { + switch { + case errors.Is(err, pgtooling.ErrDbVersionIsNewer): + // Suppose the following case: + // 1. We have a release that does database migrations. It updates versions table successfully. + // 2. Then we need to roll back the release to previous version. + // The previous version contains set of migration that is smaller than the current version. + // + // That is when we get this error. + // + // But that is OK, we don't need to error out: our migrations are always backwards compatible, + // so old version of code is supposed to work. + lg.Warn( + "database version is newer than migrations that we have, that is OK only during rollback", + zap.Error(err), + ) + default: + lg.Fatal("cannot migrate", zap.Error(err)) + } + return + } + lg.Info("successfully migrated") +} + +var ( + lockTimeout string + flagDestination string + migrationsDir string + tableName string +) + +func init() { + flag.StringVar( + &lockTimeout, + "locktimeout", + "5s", + "value for SET lock_timeout. Consider lower values to do not block others in the lock queue for long time.", + ) + flag.StringVar(&flagDestination, "destination", "last", "migration destination") + flag.StringVar(&tableName, "table", "public.schema_version", "table name with migration version") +} diff --git a/internal/rdb/config.go b/internal/rdb/config.go index 6923d7f..fd096be 100644 --- a/internal/rdb/config.go +++ b/internal/rdb/config.go @@ -37,7 +37,7 @@ func (c *Config) CheckAndSetDefaults() error { func DefaultConfig() Config { return Config{ DSN: secret.NewString( - "postgres://postgres:password@localhost:5432/postgres", + "postgres://postgres:password@telegramsearch-pg:5432/postgres", ), MaxConnLifetime: encodingtooling.Duration{Duration: time.Hour}, MaxOpenConns: 50,