Add DB-first channel support and live send tests

README.md

@@ -54,6 +54,15 @@ Easiest way is to ask for new channel at [@r_channels](https://t.me/r_channels).
 But if you are geek enough then install mongodb, ffmpeg, python and setup cron:
 
 ```cron
-46 * * * * ~/reddit2telegram/auto_update.sh
-* * * * * ~/reddit2telegram/reddit2telegram/cron_job.sh
+46 * * * * /root/reddit2telegram/auto_update.sh
+* * * * * /root/reddit2telegram/reddit2telegram/cron_thread_job.sh
+```
+
+Tests
+-----
+
+Live integration tests send real messages to `@r_channels_test` and use Reddit API.
+
+```bash
+R2T_LIVE_TESTS=1 /root/reddit2telegram/.venv/bin/python -m unittest tests/test_live_send.py
 ```

dev/housekeeping.md

@@ -8,3 +8,33 @@ Check disk usage:
 ```bash
 ncdu /
 ```
+
+## r2t_consumer_app.service
+Systemd unit for the task queue consumer.
+
+Unit file location:
+```
+/etc/systemd/system/r2t_consumer_app.service
+```
+
+Unit file contents:
+```ini
+[Unit]
+Description=r2t_consumer
+
+[Service]
+WorkingDirectory=/root/reddit2telegram/reddit2telegram
+ExecStart=/root/reddit2telegram/.venv/bin/python task_queue_consumer.py --config configs/prod.yml
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Common commands:
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable --now r2t_consumer_app.service
+sudo systemctl status --no-pager r2t_consumer_app.service
+```

reddit2telegram/channels/reddit2telegram/app.py

@@ -2,7 +2,6 @@
 
 import random
 from datetime import datetime
-import os
 import hashlib
 
 import pymongo
@@ -99,12 +98,10 @@ def what_channel(submodule_name_to_promte):
 
 
 def get_tags(submodule_name_to_promte):
-    tags_filename = os.path.join('channels', submodule_name_to_promte, 'tags.txt')
-    if not os.path.exists(tags_filename):
+    tags_string = utils.channels_stuff.get_tags_for_submodule(submodule_name_to_promte)
+    if not tags_string:
         return None
-    with open(tags_filename, 'r') as tags_file:
-        tags = tags_file.read()
-        return tags.split()
+    return tags_string.split()
 
 
 def make_nice_submission(submission, r2t, submodule_name_to_promte, extra_ending=None, **kwargs):

reddit2telegram/channels/tech_receiver/app.py

@@ -35,7 +35,7 @@ def send_post(submission, r2t):
             'last_update': 0
         })
 
-    updates = r2t.telegram_bot.get_updates(offset=last_update_doc['last_update'])
+    updates = r2t.get_updates(offset=last_update_doc['last_update'])
 
     last_update = 0
     for update in updates:
@@ -59,7 +59,7 @@ def send_post(submission, r2t):
             continue
 
         message_id = update['message']['message_id']
-        r2t.telegram_bot.forward_message(chat_id=get_dev_channel(), from_chat_id=user_id, message_id=message_id)
+        r2t.forward_message(chat_id=get_dev_channel(), from_chat_id=user_id, message_id=message_id)
         if int(update['message']['chat']['id']) == int(config['telegram']['papa']):
             # print('>>>>>>>>>>>>>>>>>^^^^^^^^^^^^^^')
             text = update['message']['text']

reddit2telegram/channels/tech_store_stat/app.py

@@ -151,11 +151,31 @@ def send_post(submission, r2t):
         'errors': 0,
         'prev_members': 0
     }
-    all_submodules = get_all_submodules()
+    all_submodules = list(get_all_submodules())
     number_of_modules = len(all_submodules)
     # sleep_coef = math.log(450 / number_of_modules) / math.log(2.718281828 / 3.14159)
     sleep_coef = 2000 / 3 / 2 / number_of_modules
     channels_stat = dict()
+    missing_channels = set()
+
+    def is_missing_channel_error(exc):
+        msg = str(exc).lower()
+        return any(key in msg for key in (
+            'chat not found',
+            'channel not found',
+            'not found',
+            'forbidden',
+            'private'
+        ))
+
+    def notify_missing_channel(channel_name):
+        if channel_name in missing_channels:
+            return
+        missing_channels.add(channel_name)
+        try:
+            r2t.send_text('Missing channel: {channel}.'.format(channel=channel_name))
+        except Exception:
+            pass
     for submodule_name in random.sample(all_submodules, k=number_of_modules):
         short_sleep(sleep_coef)
         submodule = utils.channels_stuff.import_submodule(submodule_name)
@@ -172,12 +192,20 @@ def send_post(submission, r2t):
             total['admins'] += len(admins)
         except Exception as e:
             total['errors'] += 1
-            err_to_send = 'Failed to get admins for {channel}.'.format(channel=channel_name)
-            r2t.send_text(err_to_send)
+            err_to_send = 'Failed to get admins for {channel}. {error}'.format(
+                channel=channel_name,
+                error=str(e)
+            )
+            try:
+                r2t.send_text(err_to_send)
+            except Exception:
+                pass
+            if is_missing_channel_error(e):
+                notify_missing_channel(channel_name)
             logging.error(err_to_send)
         short_sleep(sleep_coef)
         try:
-            current_members_cnt = r2t.telegram_bot.get_chat_members_count(chat_id=channel_name)
+            current_members_cnt = r2t.get_chat_members_count(chat_id=channel_name)
             stat_to_store['members_cnt'] = current_members_cnt
             total['members'] += current_members_cnt
             prev_members_cnt = get_last_members_cnt(r2t, channel_name)
@@ -190,8 +218,16 @@ def send_post(submission, r2t):
             }
         except Exception as e:
             total['errors'] += 1
-            err_to_send = 'Failed to get members count for {channel}.'.format(channel=channel_name)
-            r2t.send_text(err_to_send)
+            err_to_send = 'Failed to get members count for {channel}. {error}'.format(
+                channel=channel_name,
+                error=str(e)
+            )
+            try:
+                r2t.send_text(err_to_send)
+            except Exception:
+                pass
+            if is_missing_channel_error(e):
+                notify_missing_channel(channel_name)
             logging.error(err_to_send)
         else:
             # If they pass something special