put bot into docker and Tilt [incomplete]

.gitignore

@@ -0,0 +1,2 @@
+tmp
+.env

.tool-versions

@@ -1,2 +1,3 @@
 uv 0.6.1
-golang 1.22.12
+golang 1.23.2
+tilt 0.33.21

CONTRIBUTING.md

@@ -11,9 +11,27 @@ Add yourself to http://62.84.113.17/ to get notified about changes.
 
 This describes how to run the repo locally.
 
-## Prerequisites
+### Prerequisites
 
 This repository is using [.tool-versions](https://asdf-vm.com/manage/configuration.html). [Mise](https://github.com/jdx/mise) is recommended for managing the versions of the tools.
 Integrate it with your shell and you should have all the needed tools installed.
 By cd in the root of the repository, mise will download and install the needed versions of the tools, except for docker-compose and docker. 
 
+Why docker-compose and docker are not included in .tool-versions? They are heavy for the system, many options to have
+them exists (podman, orbstack, colima, docker, etc, etc). You need to configure them yourself.
+
+This uses:
+
+* [tilt-dev](https://tilt.dev/)
+* docker-compose
+
+### Run the local setup
+
+The following command must be enough to run minimal setup:
+
+```bash
+tilt up
+```
+
+If you see warnings in the console about missing environment keys,
+that means your setup is not complete. You need to provide the missing keys in `.env` file.

Tiltfile

@@ -0,0 +1,98 @@
+# -*- mode: Python -*-
+
+#
+# Command line args
+#
+
+# Specifies a list of args like `tilt up foo bar blabla`
+config.define_string_list("to-run", args=True)
+cfg = config.parse()
+
+
+TILT_MODE_DEFAULT = "default"
+
+# minimal subset of resources to develop
+default_resources = [
+  "telegramsearch",
+  "telegramsearch-pg",
+
+]
+
+groups = {
+    TILT_MODE_DEFAULT: default_resources,
+}
+
+# Resources to run, empty array means everything
+resources = []
+
+# Run TILT_MODE_DEFAULT by default
+tilt_args = cfg.get("to-run", [TILT_MODE_DEFAULT])
+
+for arg in tilt_args:
+    if arg in groups:
+        resources += groups[arg]
+    else:
+        # Also support specifying individual services instead of groups,
+        # e.g. `tilt up a b d`
+        resources.append(arg)
+
+
+# Tells Tilt to only run specified resources
+config.set_enabled_resources(resources)
+
+#
+# Plugins
+#
+
+load("ext://uibutton", "cmd_button")
+
+#
+# Docker compose
+#
+
+docker_compose_files = [
+    "./docker-compose.yml",
+    "./docker-compose.persistent.yml",
+]
+docker_compose(docker_compose_files)
+
+#
+# Labels
+#
+
+dc_resource(
+    "telegramsearch",
+    labels=["bot"],
+    resource_deps=[
+      "telegramsearch-pg",
+    ],
+)
+
+dc_resource(
+    "telegramsearch-pg",
+    labels=["bot"],
+)
+
+local_resource(
+  "migrate telegramsearch-pg",
+    cmd="DOCKER_BUILDKIT=1 docker build . -f app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile -t yanakipre/telegramsearch-migrations:local && docker run --network yanakipre_net --env DATABASE_URL='postgres://postgres:password@10.30.41.52:5432/telegramsearch' yanakipre/telegramsearch-migrations:local",
+    labels=["bot"],
+    auto_init=False,
+    trigger_mode=TRIGGER_MODE_MANUAL,
+    resource_deps=[
+      "telegramsearch-pg",
+    ],
+)
+
+docker_build(
+    "yanakipre/telegramsearch:local",
+    context=".",
+    dockerfile="app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile",
+    only=[
+        "app/telegramsearch",
+        "internal",
+        "go.mod",
+        "go.sum",
+    ],
+)
+

app/archwg-router/.gitignore

@@ -0,0 +1 @@
+.venv

app/telegramsearch/cmd/telegramsearch/telegramsearch.Dockerfile

@@ -0,0 +1,86 @@
+# Use BuildKit’s Dockerfile frontend to cache local builds.
+# https://www.docker.com/blog/containerize-your-go-developer-environment-part-2/
+#
+# Run docker with DOCKER_BUILDKIT=1 prepended to docker command to activate Build Kit backend.
+#
+# On linux
+# cat /etc/docker/daemon.json
+#{ "features": { "buildkit": true } }
+# syntax = docker/dockerfile:1-experimental
+
+
+#
+# Telegram bot image
+#
+
+#
+# Intermediate image to build golang API server
+#
+
+# Image version version is sha256 to speed up builds (especially local ones) and keep builds reproducible.
+# 1.23.2-bullseye
+FROM golang@sha256:ecb3fe70e1fd6cef4c5c74246a7525c3b7d59c48ea0589bbb0e57b1b37321fb9 AS go-build
+ARG DEBUG=false
+
+# define parent caching directory
+ENV CACHE=/cache
+
+# define go caching directories
+ENV GOCACHE=$CACHE/gocache
+ENV GOMODCACHE=$CACHE/gomodcache
+
+WORKDIR /build
+
+# cache dependencies before building
+RUN --mount=type=cache,target=$CACHE \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    go mod download -x
+
+
+RUN mkdir /debugbin; \
+    if [ "$DEBUG" = "true" ]; then \
+    GOBIN=/debugbin/ go install github.com/go-delve/delve/cmd/dlv@v1.23.1; \
+    fi
+
+RUN apt-get update && apt-get install -y git
+
+# build executable, store in cache so rebuilds can no-op
+RUN --mount=type=cache,target=$CACHE \
+    --mount=type=bind,target=. \
+    if [ "$DEBUG" = "true" ]; then \
+      go build -buildvcs=auto -o $CACHE/telegramsearch -gcflags="all=-N -l" ./app/telegramsearch/cmd/telegramsearch && cp $CACHE/telegramsearch /bin; \
+    else \
+      go build -buildvcs=auto -o $CACHE/telegramsearch ./app/telegramsearch/cmd/telegramsearch && cp $CACHE/telegramsearch /bin; \
+    fi
+# IIUC, there is no such thing as release build in Go. It bundles
+# debug symbols by default. We can strip them with `go build -ldflags "-s -w"`,
+# but why may want that?
+
+#
+# Final image to be exported
+#
+# Image version is fixed to speed up builds (especially local ones) and keep builds reproducible.
+# bullseye-slim
+FROM debian@sha256:60a596681410bd31a48e5975806a24cd78328f3fd6b9ee5bc64dca6d46a51f29
+
+# it's an antipattern to run the next lines in multiple steps,
+# but we're having frequent 4294967295 exit code and this is an attempt to debug it.
+# after it's solved - combine the steps into single one.
+RUN apt-get update || cat /var/log/apt/*.log
+RUN apt-get install --yes --no-install-recommends \
+    ca-certificates \
+    openssl || cat /var/log/apt/*.log
+RUN apt-get clean || cat /var/log/apt/*.log
+
+RUN useradd -d /example yanakipre
+USER yanakipre
+WORKDIR /yanakipre
+
+# Only copy if the /debugbin/dlv exists, otherwise it will do nothing
+COPY --chown=yanakipre --from=go-build /debugbin/dlv* /usr/local/bin/
+COPY --from=go-build /bin/telegramsearch /yanakipre/telegramsearch
+
+EXPOSE 3000 9090
+ENTRYPOINT ["/yanakipre/telegramsearch"]
+CMD [""]

app/telegramsearch/cmd/telegramsearch/telegramsearch.yaml.example

@@ -0,0 +1,68 @@
+ctlv1:
+  help_text: |
+    Добрый день! Это - бот, который поможет вам найти ответы на ваши вопросы.
+
+    Чтобы задать вопрос, просто напишите его в чат. Он постарается найти наиболее подходящий ответ на ваш вопрос.
+    Бот не выдумывает от себя, все его данные собраны от живых людей.
+
+    Я, разработчик, буду очень признателен, если вы поделитесь своими впечатлениями о боте и порекомендуете его своим друзьям, если он вам полезен.
+
+    Вот тут можно связаться с разработчиком: https://substack.com/home/post/p-148053843
+  news_text: |
+    Новости:
+
+    Мы открыли чат, в котором все вопросы можно задавать и боту, и людям. Присоединяйтесь: https://t.me/+8trW_-0GEFI1NTE0
+    Мы планируем добавить в бота функцию показа ссылок на чаты, где было найдена эта информация.
+    Это позволит вам быстро найти чаты, где обсуждаются интересные вам темы.
+    Если вам есть что прокомментировать или добавить - пишите в чате https://t.me/+8trW_-0GEFI1NTE0 или в https://substack.com/home/post/p-148053843
+  no_explained_answer: К сожалению, у меня нет информации об этом сообщении. Возможно
+    оно было задано слишком давно и я о нем забыл.
+  noresultsanswer: К сожалению, у меня недостаточно информации чтобы ответить на данный
+    вопрос. Но я учусь и, вероятно, смогу ответить на него позже.
+  staleresponsestext: В ответе не использовано информации свежее чем от %s
+  freshresponsestext: 'Обсуждений: %d'
+  stalethreshold: 17520h0m0s
+postgres_rw:
+  rdb:
+    search_path: ""
+    database_type: ""
+    database_url: xxx
+    max_conn_lifetime: 1h0m0s
+    max_open_conns: 50
+    collect_metrics_interval: 5s
+openai:
+  embeddingconfig:
+    model: text-embedding-3-small
+  apikey: xxx
+  transport:
+    client_name: openapi
+    timeout: 0s
+    dial_timeout: 1m0s
+    dial_keep_alive: 5m0s
+    idle_conn_timeout: 3m0s
+    tls_handshake_timeout: 10s
+    expect_continue_timeout: 1s
+    response_header_timeout: 1m0s
+    disable_keep_alives: false
+    max_idle_conns: 100
+    max_idle_conns_per_host: 2
+    max_conns_per_host: 0
+  retries:
+    backoff: 100ms
+    attempts: 10
+  rate_limiters: []
+  asking_about: Cyprus
+  donothighlight: Cyprus
+logging:
+  sink: stdout
+  log_level: DEBUG
+  log_format: json
+  filters:
+    by_logger_name: []
+    by_exact_name: []
+telegram_transport:
+  token: xxx
+  greeting: ""
+telegram_v2:
+  appid: xxx
+  apphash: xxx

app/telegramsearch/internal/pkg/staticconfig/loader.go

@@ -4,20 +4,23 @@ import (
 	"context"
 	"errors"
 	"fmt"
-
 	"github.com/heetch/confita"
 	"github.com/heetch/confita/backend"
 )
 
 // Backend specifies custom logic for config loading
 type Backend struct{}
 
-func (b Backend) Unmarshal(ctx context.Context, to any) error {
-	_, ok := to.(*Config)
+func (b Backend) Unmarshal(_ context.Context, to any) error {
+	cfg, ok := to.(*Config)
 	if !ok {
 		return fmt.Errorf("cannot unmarshall to Config: %+v", to)
 	}
 
+	cfg.PostgresRW.RDB.DSN.FromEnv("DATABASE_URL")
+	cfg.OpenAI.ApiKey.FromEnv("OPENAI_API_KEY")
+	cfg.TelegramTransport.Token.FromEnv("TELEGRAM_BOT_TOKEN")
+
 	return nil
 }
 

app/telegramsearch/telegramsearch-db/generate.go

@@ -1,3 +1,3 @@
 package telegramsearchdb
 
-//go:generate go run ../db-management/cmd/db-freeze --dbpath=.
+//go:generate go run ../../../internal/db-management/cmd/db-freeze --dbpath=.

app/telegramsearch/telegramsearch-db/telegramsearch-db.Dockerfile

@@ -0,0 +1,41 @@
+# Image version version is sha256 to speed up builds (especially local ones) and keep builds reproducible.
+# 1.23.2-alpine
+FROM golang@sha256:9dd2625a1ff2859b8d8b01d8f7822c0f528942fe56cfe7a1e7c38d3b8d72d679 AS build
+
+
+WORKDIR /build
+
+COPY go.mod .
+COPY go.sum .
+
+# define parent caching directory
+ENV CACHE=/cache
+
+# define go caching directories
+ENV GOCACHE=$CACHE/gocache
+ENV GOMODCACHE=$CACHE/gomodcache
+
+# cache dependencies before building
+RUN --mount=type=cache,target=$CACHE \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    go mod download -x
+
+RUN --mount=type=cache,target=$CACHE \
+    --mount=type=bind,target=. \
+    go build -o $CACHE/postgres-migrate ./internal/db-management/cmd/postgres-migrate && cp $CACHE/postgres-migrate /bin/
+
+# Image version is fixed to speed up builds (especially local ones) and keep builds reproducible.
+# 3.14.10
+FROM alpine@sha256:0f2d5c38dd7a4f4f733e688e3a6733cb5ab1ac6e3cb4603a5dd564e5bfb80eed
+
+RUN apk add --no-cache bash ca-certificates
+
+# https://github.com/moby/moby/issues/30081#issuecomment-1137512170
+RUN mkdir -p /db
+WORKDIR /db
+
+COPY --from=build /bin/postgres-migrate /usr/local/bin/postgres-migrate
+COPY app/telegramsearch/telegramsearch-app/migrations /db/migrations
+
+CMD ["postgres-migrate"]