From d13ebf9530d1ac101bbde707880a365884eac221 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:00:07 +1200 Subject: [PATCH 001/109] refactor: renaming module --- {tasks_queue => django_leek}/API.py | 0 {tasks_queue => django_leek}/__init__.py | 0 {tasks_queue => django_leek}/app_settings.py | 0 {tasks_queue => django_leek}/helpers.py | 0 {tasks_queue => django_leek}/migrations/0001_initial.py | 0 {tasks_queue => django_leek}/migrations/__init__.py | 0 {tasks_queue => django_leek}/models.py | 0 {tasks_queue => django_leek}/run_failed_tasks.py | 0 {tasks_queue => django_leek}/server.py | 0 {tasks_queue => django_leek}/service-start.py | 0 {tasks_queue => django_leek}/service-stop-worker.py | 0 {tasks_queue => django_leek}/shell.py | 0 {tasks_queue => django_leek}/task.py | 0 {tasks_queue => django_leek}/tests.py | 0 {tasks_queue => django_leek}/worker.py | 0 {tasks_queue => django_leek}/worker_manager.py | 0 16 files changed, 0 insertions(+), 0 deletions(-) rename {tasks_queue => django_leek}/API.py (100%) rename {tasks_queue => django_leek}/__init__.py (100%) rename {tasks_queue => django_leek}/app_settings.py (100%) rename {tasks_queue => django_leek}/helpers.py (100%) rename {tasks_queue => django_leek}/migrations/0001_initial.py (100%) rename {tasks_queue => django_leek}/migrations/__init__.py (100%) rename {tasks_queue => django_leek}/models.py (100%) rename {tasks_queue => django_leek}/run_failed_tasks.py (100%) rename {tasks_queue => django_leek}/server.py (100%) rename {tasks_queue => django_leek}/service-start.py (100%) rename {tasks_queue => django_leek}/service-stop-worker.py (100%) rename {tasks_queue => django_leek}/shell.py (100%) rename {tasks_queue => django_leek}/task.py (100%) rename {tasks_queue => django_leek}/tests.py (100%) rename {tasks_queue => django_leek}/worker.py (100%) rename {tasks_queue => django_leek}/worker_manager.py (100%) diff --git a/tasks_queue/API.py b/django_leek/API.py similarity index 100% rename from tasks_queue/API.py rename to django_leek/API.py diff --git a/tasks_queue/__init__.py b/django_leek/__init__.py similarity index 100% rename from tasks_queue/__init__.py rename to django_leek/__init__.py diff --git a/tasks_queue/app_settings.py b/django_leek/app_settings.py similarity index 100% rename from tasks_queue/app_settings.py rename to django_leek/app_settings.py diff --git a/tasks_queue/helpers.py b/django_leek/helpers.py similarity index 100% rename from tasks_queue/helpers.py rename to django_leek/helpers.py diff --git a/tasks_queue/migrations/0001_initial.py b/django_leek/migrations/0001_initial.py similarity index 100% rename from tasks_queue/migrations/0001_initial.py rename to django_leek/migrations/0001_initial.py diff --git a/tasks_queue/migrations/__init__.py b/django_leek/migrations/__init__.py similarity index 100% rename from tasks_queue/migrations/__init__.py rename to django_leek/migrations/__init__.py diff --git a/tasks_queue/models.py b/django_leek/models.py similarity index 100% rename from tasks_queue/models.py rename to django_leek/models.py diff --git a/tasks_queue/run_failed_tasks.py b/django_leek/run_failed_tasks.py similarity index 100% rename from tasks_queue/run_failed_tasks.py rename to django_leek/run_failed_tasks.py diff --git a/tasks_queue/server.py b/django_leek/server.py similarity index 100% rename from tasks_queue/server.py rename to django_leek/server.py diff --git a/tasks_queue/service-start.py b/django_leek/service-start.py similarity index 100% rename from tasks_queue/service-start.py rename to django_leek/service-start.py diff --git a/tasks_queue/service-stop-worker.py b/django_leek/service-stop-worker.py similarity index 100% rename from tasks_queue/service-stop-worker.py rename to django_leek/service-stop-worker.py diff --git a/tasks_queue/shell.py b/django_leek/shell.py similarity index 100% rename from tasks_queue/shell.py rename to django_leek/shell.py diff --git a/tasks_queue/task.py b/django_leek/task.py similarity index 100% rename from tasks_queue/task.py rename to django_leek/task.py diff --git a/tasks_queue/tests.py b/django_leek/tests.py similarity index 100% rename from tasks_queue/tests.py rename to django_leek/tests.py diff --git a/tasks_queue/worker.py b/django_leek/worker.py similarity index 100% rename from tasks_queue/worker.py rename to django_leek/worker.py diff --git a/tasks_queue/worker_manager.py b/django_leek/worker_manager.py similarity index 100% rename from tasks_queue/worker_manager.py rename to django_leek/worker_manager.py From b1a8d80da9ca95aadb24eff45e17b429937f8ac5 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:07:37 +1200 Subject: [PATCH 002/109] ex: clean up gitignore --- .gitignore | 58 +----------------------------------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index 528730d..70024f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,63 +1,7 @@ -# Wing IDE -*.wpr -*.wpu - - - -# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -#lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +/venv/ # Unit test / coverage reports -htmlcov/ -.tox/ .coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ From 304fe5e09b675d34d65d4d7ca502beaf56f69067 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:07:46 +1200 Subject: [PATCH 003/109] ex: vs code support --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..5b80df3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "venv/bin/python" +} \ No newline at end of file From 5110fe14517b519c356690569577e0514b3f6a80 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:18:47 +1200 Subject: [PATCH 004/109] create .whl file --- .gitignore | 5 +++++ setup.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 70024f3..2fe87c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ __pycache__/ # Unit test / coverage reports .coverage + +# setuptools +/build/ +/dist/*.whl +*.egg-info/ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f2f8caf --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +import os +from setuptools import find_packages, setup + +# allow setup.py to be run from any path +os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + + +with open('README.md') as f: + README = f.read() + + +setup( + name='django-leek', + version='0.1', + packages=find_packages(), + include_package_data=True, + license='MIT License', + description='A simple Django app to offload tasks from main web server', + long_description=README, + long_description_content_type='text/markdown', + url='https://github.com/Volumental/django-leek', + author='Samuel Carlsson', + author_email='samuel.carlsson@volumental.com', + classifiers=[ + 'Environment :: Web Environment', + 'Framework :: Django', + 'Framework :: Django :: 1.9', # replace "X.Y" as appropriate + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', # example license + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + ], +) \ No newline at end of file From 8247f709983d3fdbe0680e9f2683b2921e123227 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:18:51 +1200 Subject: [PATCH 005/109] requirements --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c5c5dd6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +django==1.11 +pylint From 5e110457b1aceba89bd716e626a3422951f908b6 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 16:43:52 +1200 Subject: [PATCH 006/109] Minimal test app --- run_test_app.sh | 4 ++++ test_app/README.md | 2 ++ test_app/__init__.py | 0 test_app/settings.py | 17 +++++++++++++++++ test_app/urls.py | 6 ++++++ test_app/views.py | 4 ++++ 6 files changed, 33 insertions(+) create mode 100755 run_test_app.sh create mode 100644 test_app/README.md create mode 100644 test_app/__init__.py create mode 100644 test_app/settings.py create mode 100644 test_app/urls.py create mode 100644 test_app/views.py diff --git a/run_test_app.sh b/run_test_app.sh new file mode 100755 index 0000000..a512e03 --- /dev/null +++ b/run_test_app.sh @@ -0,0 +1,4 @@ +#!/bin/sh +. venv/bin/activate +django-admin migrate --pythonpath=. --settings=test_app.settings +django-admin runserver --pythonpath=. --settings=test_app.settings diff --git a/test_app/README.md b/test_app/README.md new file mode 100644 index 0000000..734ccc1 --- /dev/null +++ b/test_app/README.md @@ -0,0 +1,2 @@ +# Test App +Minimal django app that depend on the django_leek module diff --git a/test_app/__init__.py b/test_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_app/settings.py b/test_app/settings.py new file mode 100644 index 0000000..0d9e683 --- /dev/null +++ b/test_app/settings.py @@ -0,0 +1,17 @@ +ALLOWED_HOSTS = ['localhost', '127.0.0.1'] +SECRET_KEY = "not so secret" + +DEBUG=True + +INSTALLED_APPS = [ + 'django_leek' +] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'db.sqlite3', + } +} + +ROOT_URLCONF = 'test_app.urls' diff --git a/test_app/urls.py b/test_app/urls.py new file mode 100644 index 0000000..be4d940 --- /dev/null +++ b/test_app/urls.py @@ -0,0 +1,6 @@ +from django.conf.urls import url +from test_app.views import * + +urlpatterns = [ + url(r'^$', index), +] diff --git a/test_app/views.py b/test_app/views.py new file mode 100644 index 0000000..fddc0ce --- /dev/null +++ b/test_app/views.py @@ -0,0 +1,4 @@ +from django.http import HttpResponse + +def index(request): + return HttpResponse('Test app', content_type='text/plain') From 0849c2dcbb43404f52da71f4ddc732d2160d5b9c Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 17:26:27 +1200 Subject: [PATCH 007/109] test_app: queue task --- .gitignore | 3 +++ test_app/app.py | 21 +++++++++++++++++++++ test_app/settings.py | 9 ++++++++- test_app/templates/index.html | 10 ++++++++++ test_app/urls.py | 6 ------ test_app/views.py | 4 ---- 6 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 test_app/app.py create mode 100644 test_app/templates/index.html delete mode 100644 test_app/urls.py delete mode 100644 test_app/views.py diff --git a/.gitignore b/.gitignore index 2fe87c4..be7ca2c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ __pycache__/ /build/ /dist/*.whl *.egg-info/ + +# test app +db.sqlite3 diff --git a/test_app/app.py b/test_app/app.py new file mode 100644 index 0000000..7f21609 --- /dev/null +++ b/test_app/app.py @@ -0,0 +1,21 @@ +from django.conf.urls import url +from django.http import HttpResponse +from django.shortcuts import render + +from django_leek.API import push_task_to_queue + +def task(): + print('Executed on queue!') + + +def index(request): + if 'queue' in request.GET: + push_task_to_queue(task) + return render(request, 'index.html', {'message': '✓ task queued'}) + + return render(request, 'index.html') + + +urlpatterns = [ + url(r'^$', index), +] diff --git a/test_app/settings.py b/test_app/settings.py index 0d9e683..7cc4029 100644 --- a/test_app/settings.py +++ b/test_app/settings.py @@ -1,3 +1,4 @@ +import os ALLOWED_HOSTS = ['localhost', '127.0.0.1'] SECRET_KEY = "not so secret" @@ -14,4 +15,10 @@ } } -ROOT_URLCONF = 'test_app.urls' +ROOT_URLCONF = 'test_app.app' + +PROJECT_PATH = os.path.realpath(os.path.dirname(__file__)) +TEMPLATES = [{ + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ os.path.join(PROJECT_PATH, 'templates/')]} +] diff --git a/test_app/templates/index.html b/test_app/templates/index.html new file mode 100644 index 0000000..63eb3ff --- /dev/null +++ b/test_app/templates/index.html @@ -0,0 +1,10 @@ + + + +

{{ message }}

+
+ + +
+ + \ No newline at end of file diff --git a/test_app/urls.py b/test_app/urls.py deleted file mode 100644 index be4d940..0000000 --- a/test_app/urls.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.conf.urls import url -from test_app.views import * - -urlpatterns = [ - url(r'^$', index), -] diff --git a/test_app/views.py b/test_app/views.py deleted file mode 100644 index fddc0ce..0000000 --- a/test_app/views.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.http import HttpResponse - -def index(request): - return HttpResponse('Test app', content_type='text/plain') From c6f98e9ef2fcab0d43ea00174d5a704b9dd4e7e8 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 17:26:40 +1200 Subject: [PATCH 008/109] dist: don't distribute test_app --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f2f8caf..e5348e7 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='django-leek', version='0.1', - packages=find_packages(), + packages=['django_leek'], include_package_data=True, license='MIT License', description='A simple Django app to offload tasks from main web server', From 1e8cfd429aa45793f0a1c39e846f244549e52a09 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 17:26:49 +1200 Subject: [PATCH 009/109] python3.5+ --- django_leek/helpers.py | 6 +++--- django_leek/run_failed_tasks.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index 89ea0bf..ea9ab13 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -1,5 +1,5 @@ import datetime -import cPickle +import pickle import base64 from . import models @@ -7,13 +7,13 @@ def unpack(pickled_task): - new_task = cPickle.loads(base64.b64decode(pickled_task)) + new_task = pickle.loads(base64.b64decode(pickled_task)) assert isinstance(new_task,Task) return new_task def serielize(task): - return base64.b64encode(cPickle.dumps(task)) + return base64.b64encode(pickle.dumps(task)) def save_task_to_db(new_task): diff --git a/django_leek/run_failed_tasks.py b/django_leek/run_failed_tasks.py index de3a568..204c898 100644 --- a/django_leek/run_failed_tasks.py +++ b/django_leek/run_failed_tasks.py @@ -1,4 +1,4 @@ -import cPickle +import pickle import base64 import os import sys @@ -16,7 +16,7 @@ Lfailed_tasks_id = FailedTasks.objects.values_list("task_id",flat=True) tasks = QueuedTasks.objects.filter(pk__in=Lfailed_tasks_id) for r in tasks: - task = cPickle.loads(base64.b64decode(r.pickled_task)) + task = pickle.loads(base64.b64decode(r.pickled_task)) task.run() From ac250ec6e7b4d7c40f813d065de08acb4dad0e5f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 7 May 2018 17:35:40 +1200 Subject: [PATCH 010/109] runleek command --- django_leek/management/__init__.py | 0 django_leek/management/commands/__init__.py | 0 django_leek/management/commands/runleek.py | 19 +++++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 django_leek/management/__init__.py create mode 100644 django_leek/management/commands/__init__.py create mode 100644 django_leek/management/commands/runleek.py diff --git a/django_leek/management/__init__.py b/django_leek/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_leek/management/commands/__init__.py b/django_leek/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_leek/management/commands/runleek.py b/django_leek/management/commands/runleek.py new file mode 100644 index 0000000..d98bc86 --- /dev/null +++ b/django_leek/management/commands/runleek.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand, CommandError + + +class Command(BaseCommand): + help = 'Starts leek worker server' + + #def add_arguments(self, parser): + # parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + from django_leek import worker_manager + from django_leek.server import TaskSocketServerThread + import time + + worker_manager.start() + server_thread = TaskSocketServerThread('localhost', 8002) + time.sleep(5) + socket_server = server_thread.socket_server() + socket_server.serve_forever() From 5a5aae6808a0824cec756b2f81e1bcc3f2427cd5 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 10:44:02 +1200 Subject: [PATCH 011/109] 2to3: module names --- django_leek/server.py | 7 ++++--- django_leek/worker.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 55d4291..27fd758 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,5 +1,6 @@ -import SocketServer +import socketserver import threading + from . import worker_manager Dcommands = { @@ -9,7 +10,7 @@ 'stop':worker_manager.stop } -class TaskSocketServer(SocketServer.BaseRequestHandler): +class TaskSocketServer(socketserver.BaseRequestHandler): def handle(self): @@ -58,7 +59,7 @@ def socket_server(self): def run(self): - self.server = SocketServer.TCPServer((self.host,self.port), TaskSocketServer) + self.server = socketserver.TCPServer((self.host,self.port), TaskSocketServer) diff --git a/django_leek/worker.py b/django_leek/worker.py index a62f217..cb22601 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -3,7 +3,7 @@ import os import sys import threading -import Queue +import queue # environ settings variable, should be the same as in manage.py os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") @@ -16,10 +16,10 @@ class Worker(threading.Thread): def __init__(self,logger_name=None): - threading.Thread.__init__(self, name="django-tasks-queue") + threading.Thread.__init__(self, name="django-leek") self._stopevent = threading.Event() self.setDaemon(1) - self.worker_queue = Queue.Queue() + self.worker_queue = queue.Queue() self.tasks_counter = 0 if logger_name != None: self.logger = logging.getLogger(logger_name) From e18d794bf0aecc9d607ac27c8b7d3529e9a984a9 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 10:45:45 +1200 Subject: [PATCH 012/109] 2to3: no `message` attribute on `Exception` --- django_leek/helpers.py | 2 +- django_leek/server.py | 8 ++++---- django_leek/worker.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index ea9ab13..ca4a6c4 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -26,7 +26,7 @@ def save_task_to_db(new_task): def save_task_failed(task,exception): - t = models.FailedTasks(task_id=task.db_id,exception=exception.message) + t = models.FailedTasks(task_id=task.db_id,exception=str(exception)) t.save() diff --git a/django_leek/server.py b/django_leek/server.py index 27fd758..817df55 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -17,7 +17,7 @@ def handle(self): try: data = self.request.recv(5000).strip() #like the pickled task field except Exception as e: - response = (False,"SocketServer:%s"%e.message) + response = (False,"SocketServer:%s"%str(e)) self.request.send(response) @@ -29,18 +29,18 @@ def handle(self): else: response = (True,worker_response,) except Exception as e: - response = (False,"TaskServer Command: %s"%e.message,) + response = (False,"TaskServer Command: %s"%str(e),) else: try: worker_response = worker_manager.put_task(data) #a tuple response = worker_response except Exception as e: - response = (False,"TaskServer Put: %s"%e.message,) + response = (False,"TaskServer Put: %s"%str(e),) try: self.request.send(str(response)) except Exception as e: - self.request.send("SocketServer Response:%s"%e.message) + self.request.send("SocketServer Response:%s"%str(e)) class TaskSocketServerThread(threading.Thread): diff --git a/django_leek/worker.py b/django_leek/worker.py index cb22601..9f919b3 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -36,7 +36,7 @@ def put_task_on_queue(self,new_pickled_task): self.worker_queue.put(new_task) return True,"sent" except Exception as e: - return False,"Worker: %s"%e.message + return False,"Worker: %s"%str(e) def run_task(self,task): From 77184f1513e0b67fdd949a89b24da337432cdd4e Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 10:48:19 +1200 Subject: [PATCH 013/109] 2to3: `send` needs `bytes` --- django_leek/server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 817df55..e78f2ab 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -17,7 +17,7 @@ def handle(self): try: data = self.request.recv(5000).strip() #like the pickled task field except Exception as e: - response = (False,"SocketServer:%s"%str(e)) + response = (False, "SocketServer: {}".format(e).encode()) self.request.send(response) @@ -25,22 +25,22 @@ def handle(self): try: worker_response = Dcommands[data]() if worker_response == 'Worker Off': - response = (False,worker_response) + response = (False,worker_response.encode()) else: - response = (True,worker_response,) + response = (True,worker_response.encode(),) except Exception as e: - response = (False,"TaskServer Command: %s"%str(e),) + response = (False,"TaskServer Command: {}".format(e).encode(),) else: try: worker_response = worker_manager.put_task(data) #a tuple response = worker_response except Exception as e: - response = (False,"TaskServer Put: %s"%str(e),) + response = (False,"TaskServer Put: {}".format(e).encode(),) try: self.request.send(str(response)) except Exception as e: - self.request.send("SocketServer Response:%s"%str(e)) + self.request.send("SocketServer Response: {}".format(e).encode()) class TaskSocketServerThread(threading.Thread): From 41eb32ab71a34c11658b4082bf333484e46f72d7 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 10:59:01 +1200 Subject: [PATCH 014/109] refactor: whitespace --- django_leek/API.py | 1 - django_leek/app_settings.py | 9 +++---- django_leek/helpers.py | 18 ++++--------- django_leek/models.py | 15 ++++------- django_leek/run_failed_tasks.py | 5 ++-- django_leek/server.py | 41 ++++++++++++------------------ django_leek/service-start.py | 4 +-- django_leek/service-stop-worker.py | 7 ++--- django_leek/shell.py | 8 +++--- django_leek/task.py | 8 ++---- django_leek/worker.py | 34 ++++++++++++------------- django_leek/worker_manager.py | 16 ++++++------ 12 files changed, 67 insertions(+), 99 deletions(-) diff --git a/django_leek/API.py b/django_leek/API.py index f5e5a42..3692544 100644 --- a/django_leek/API.py +++ b/django_leek/API.py @@ -5,7 +5,6 @@ def push_task_to_queue(a_callable,*args,**kwargs): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_task = Task(a_callable,*args,**kwargs) new_task = helpers.save_task_to_db(new_task) #returns with db_id diff --git a/django_leek/app_settings.py b/django_leek/app_settings.py index 6c94404..9b36c41 100644 --- a/django_leek/app_settings.py +++ b/django_leek/app_settings.py @@ -2,18 +2,17 @@ D = { - "MAX_RETRIES":3, - "TASKS_HOST":"localhost", - "TASKS_PORT":8002 + "MAX_RETRIES": 3, + "TASKS_HOST": "localhost", + "TASKS_PORT": 8002 } if hasattr(settings,"TASKS_QUEUE"): - for key,value in getattr(settings,"TASKS_QUEUE"): + for key,value in getattr(settings, "TASKS_QUEUE"): D[key] = value MAX_RETRIES = D["MAX_RETRIES"] TASKS_HOST = D['TASKS_HOST'] TASKS_PORT = D['TASKS_PORT'] - diff --git a/django_leek/helpers.py b/django_leek/helpers.py index ca4a6c4..8bda2ef 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -5,18 +5,18 @@ from . import models from .task import Task + def unpack(pickled_task): - new_task = pickle.loads(base64.b64decode(pickled_task)) assert isinstance(new_task,Task) return new_task - + + def serielize(task): - return base64.b64encode(pickle.dumps(task)) + def save_task_to_db(new_task): - t = models.QueuedTasks() pickled_task = serielize(new_task) t = models.QueuedTasks(pickled_task=pickled_task) @@ -25,18 +25,10 @@ def save_task_to_db(new_task): return new_task def save_task_failed(task,exception): - - t = models.FailedTasks(task_id=task.db_id,exception=str(exception)) + t = models.FailedTasks(task_id=task.db_id, exception=str(exception)) t.save() def save_task_success(task): - t = models.SuccessTasks(task_id=task.db_id) t.save() - - - - - - \ No newline at end of file diff --git a/django_leek/models.py b/django_leek/models.py index 0da3b17..e6ed0e8 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -1,22 +1,17 @@ from django.db import models + class QueuedTasks(models.Model): - - pickled_task = models.CharField(max_length=5000) #max row 65535 + pickled_task = models.CharField(max_length=5000) #max row 65535 queued_on = models.DateTimeField(auto_now_add=True) - + + class SuccessTasks(models.Model): - task_id = models.IntegerField() saved_on = models.DateTimeField(auto_now_add=True) - + class FailedTasks(models.Model): - task_id = models.IntegerField() exception = models.CharField(max_length=2048) saved_on = models.DateTimeField(auto_now_add=True) - - - - \ No newline at end of file diff --git a/django_leek/run_failed_tasks.py b/django_leek/run_failed_tasks.py index 204c898..4530f8e 100644 --- a/django_leek/run_failed_tasks.py +++ b/django_leek/run_failed_tasks.py @@ -13,10 +13,9 @@ # edit to the correct mysite.tasks_queue path from mysite.tasks_queue.models import FailedTasks,QueuedTasks -Lfailed_tasks_id = FailedTasks.objects.values_list("task_id",flat=True) + +Lfailed_tasks_id = FailedTasks.objects.values_list("task_id", flat=True) tasks = QueuedTasks.objects.filter(pk__in=Lfailed_tasks_id) for r in tasks: task = pickle.loads(base64.b64decode(r.pickled_task)) task.run() - - diff --git a/django_leek/server.py b/django_leek/server.py index e78f2ab..92dfc42 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -3,64 +3,55 @@ from . import worker_manager + Dcommands = { - 'ping':worker_manager.ping, - 'waiting':worker_manager.waiting, - 'handled':worker_manager.hanled, - 'stop':worker_manager.stop + 'ping': worker_manager.ping, + 'waiting': worker_manager.waiting, + 'handled': worker_manager.hanled, + 'stop': worker_manager.stop } + class TaskSocketServer(socketserver.BaseRequestHandler): - def handle(self): - try: data = self.request.recv(5000).strip() #like the pickled task field except Exception as e: response = (False, "SocketServer: {}".format(e).encode()) self.request.send(response) - - + if data in Dcommands.keys(): try: worker_response = Dcommands[data]() if worker_response == 'Worker Off': - response = (False,worker_response.encode()) + response = (False, worker_response.encode()) else: - response = (True,worker_response.encode(),) + response = (True, worker_response.encode(),) except Exception as e: - response = (False,"TaskServer Command: {}".format(e).encode(),) - else: + response = (False, "TaskServer Command: {}".format(e).encode(),) + else: try: worker_response = worker_manager.put_task(data) #a tuple response = worker_response except Exception as e: - response = (False,"TaskServer Put: {}".format(e).encode(),) - - try: + response = (False, "TaskServer Put: {}".format(e).encode(),) + + try: self.request.send(str(response)) except Exception as e: self.request.send("SocketServer Response: {}".format(e).encode()) class TaskSocketServerThread(threading.Thread): - def __init__(self,host,port): - threading.Thread.__init__(self, name='tasks-socket-server') self.host = host self.port = port self.setDaemon(1) self.start() - + def socket_server(self): return self.server - def run(self): - - self.server = socketserver.TCPServer((self.host,self.port), TaskSocketServer) - - - - \ No newline at end of file + self.server = socketserver.TCPServer((self.host, self.port), TaskSocketServer) diff --git a/django_leek/service-start.py b/django_leek/service-start.py index 70c9457..2c02f36 100644 --- a/django_leek/service-start.py +++ b/django_leek/service-start.py @@ -6,9 +6,7 @@ import time worker_manager.start() -server_thread = TaskSocketServerThread('localhost',TASKS_PORT) +server_thread = TaskSocketServerThread('localhost', TASKS_PORT) time.sleep(5) socket_server = server_thread.socket_server() socket_server.serve_forever() - - diff --git a/django_leek/service-stop-worker.py b/django_leek/service-stop-worker.py index 978127f..c37f62f 100644 --- a/django_leek/service-stop-worker.py +++ b/django_leek/service-stop-worker.py @@ -1,8 +1,8 @@ import socket from app_settings import TASKS_HOST,TASKS_PORT -def stop_server(): +def stop_server(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((TASKS_HOST, TASKS_PORT)) sock.send("stop") @@ -10,6 +10,7 @@ def stop_server(): sock.close() print "Sent: %s" % "stop" print "Received: %s" % received - + + if __name__ == "__main__": - stop_server() \ No newline at end of file + stop_server() diff --git a/django_leek/shell.py b/django_leek/shell.py index 7386029..e9e6ee8 100644 --- a/django_leek/shell.py +++ b/django_leek/shell.py @@ -1,9 +1,9 @@ import socket import sys from app_settings import TASKS_HOST,TASKS_PORT - -def send_data(): + +def send_data(): data = sys.argv[1] sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((TASKS_HOST, TASKS_PORT)) @@ -12,7 +12,7 @@ def send_data(): sock.close() print "Sent: %s" % data print "Received: %s" % received - + + if __name__ == "__main__": send_data() - diff --git a/django_leek/task.py b/django_leek/task.py index f874320..7122389 100644 --- a/django_leek/task.py +++ b/django_leek/task.py @@ -1,9 +1,5 @@ - - class Task(object): - - def __init__(self,a_callable,*args,**kwargs): - + def __init__(self, a_callable, *args, **kwargs): assert callable(a_callable) self.task_callable = a_callable self.args = args @@ -11,4 +7,4 @@ def __init__(self,a_callable,*args,**kwargs): self.db_id = None def run(self): - self.task_callable(*self.args,**self.kwargs) + self.task_callable(*self.args, **self.kwargs) diff --git a/django_leek/worker.py b/django_leek/worker.py index 9f919b3..22f266d 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -5,17 +5,19 @@ import threading import queue + # environ settings variable, should be the same as in manage.py os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") os.environ["DJANGO_SETTINGS_MODULE"] = "mysite.settings" import django + from . import app_settings from . import helpers + class Worker(threading.Thread): - def __init__(self,logger_name=None): - + def __init__(self, logger_name=None): threading.Thread.__init__(self, name="django-leek") self._stopevent = threading.Event() self.setDaemon(1) @@ -27,19 +29,17 @@ def __init__(self,logger_name=None): self.logger = logging self.start() - - def put_task_on_queue(self,new_pickled_task): - + + def put_task_on_queue(self, new_pickled_task): try: new_task = helpers.unpack(new_pickled_task) self.tasks_counter += 1 self.worker_queue.put(new_task) - return True,"sent" + return True, "sent" except Exception as e: - return False,"Worker: %s"%str(e) - - def run_task(self,task): - + return False, "Worker: %s"%str(e) + + def run_task(self, task): for i in range(app_settings.MAX_RETRIES): try: task.run() @@ -49,7 +49,7 @@ def run_task(self,task): pass else: raise - + def stop_thread(self, timeout=None): """ Stop the thread and wait for it to end. """ if self.worker_queue != None: @@ -58,20 +58,20 @@ def stop_thread(self, timeout=None): return "Stop Set" else: return "Worker Off" - + def ping(self): if self.worker_queue != None: return "I'm OK" else: return "Worker Off" - + def status_waiting(self): return self.worker_queue.qsize() - + def status_handled(self): # all, success & failes return self.tasks_counter - + def run(self): # the code until the while statement does NOT run atomicaly # a thread while loop cycle is atomic @@ -87,8 +87,6 @@ def run(self): helpers.save_task_failed(task,e) else: helpers.save_task_success(task) - - + self.worker_queue = None self.logger.warn('Worker stopped, %s tasks handled'%self.tasks_counter) - diff --git a/django_leek/worker_manager.py b/django_leek/worker_manager.py index 896c1ee..d5f5f34 100644 --- a/django_leek/worker_manager.py +++ b/django_leek/worker_manager.py @@ -1,26 +1,26 @@ from .worker import Worker + def start(): - global worker_thread worker_thread = Worker(logger_name='main') - + + def put_task(task): - return worker_thread.put_task_on_queue(task) + def stop(): - return worker_thread.stop_thread() + def ping(): - return worker_thread.ping() + def waiting(): - return worker_thread.status_waiting() + def hanled(): - - return worker_thread.status_handled() \ No newline at end of file + return worker_thread.status_handled() From 75fa03076e5a7546a5faa6e2940accb99e2da725 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:17:39 +1200 Subject: [PATCH 015/109] ex: tests pass --- django_leek/app_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_leek/app_settings.py b/django_leek/app_settings.py index 9b36c41..7ad2567 100644 --- a/django_leek/app_settings.py +++ b/django_leek/app_settings.py @@ -1,6 +1,8 @@ from django.conf import settings +SECRET_KEY="just to make tests run" + D = { "MAX_RETRIES": 3, "TASKS_HOST": "localhost", From 5d2334b1b378da60618b8d5d504df36ae3c28901 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:19:01 +1200 Subject: [PATCH 016/109] refactor: renaming to `settings.py` --- django_leek/API.py | 2 +- django_leek/service-start.py | 2 +- django_leek/service-stop-worker.py | 2 +- django_leek/{app_settings.py => settings.py} | 0 django_leek/shell.py | 2 +- django_leek/worker.py | 6 +++--- 6 files changed, 7 insertions(+), 7 deletions(-) rename django_leek/{app_settings.py => settings.py} (100%) diff --git a/django_leek/API.py b/django_leek/API.py index 3692544..7803e64 100644 --- a/django_leek/API.py +++ b/django_leek/API.py @@ -1,7 +1,7 @@ import socket from .task import Task from . import helpers -from .app_settings import TASKS_HOST,TASKS_PORT +from .settings import TASKS_HOST,TASKS_PORT def push_task_to_queue(a_callable,*args,**kwargs): diff --git a/django_leek/service-start.py b/django_leek/service-start.py index 2c02f36..4122896 100644 --- a/django_leek/service-start.py +++ b/django_leek/service-start.py @@ -1,6 +1,6 @@ # run from shell with python -m and the full python path # to allow relative imports -from .app_settings import TASKS_PORT +from .settings import TASKS_PORT from . import worker_manager from .server import TaskSocketServerThread import time diff --git a/django_leek/service-stop-worker.py b/django_leek/service-stop-worker.py index c37f62f..7a317e0 100644 --- a/django_leek/service-stop-worker.py +++ b/django_leek/service-stop-worker.py @@ -1,5 +1,5 @@ import socket -from app_settings import TASKS_HOST,TASKS_PORT +from settings import TASKS_HOST, TASKS_PORT def stop_server(): diff --git a/django_leek/app_settings.py b/django_leek/settings.py similarity index 100% rename from django_leek/app_settings.py rename to django_leek/settings.py diff --git a/django_leek/shell.py b/django_leek/shell.py index e9e6ee8..7702f51 100644 --- a/django_leek/shell.py +++ b/django_leek/shell.py @@ -1,6 +1,6 @@ import socket import sys -from app_settings import TASKS_HOST,TASKS_PORT +from settings import TASKS_HOST, TASKS_PORT def send_data(): diff --git a/django_leek/worker.py b/django_leek/worker.py index 22f266d..f64b5ec 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -12,7 +12,7 @@ import django -from . import app_settings +from . import settings from . import helpers @@ -40,12 +40,12 @@ def put_task_on_queue(self, new_pickled_task): return False, "Worker: %s"%str(e) def run_task(self, task): - for i in range(app_settings.MAX_RETRIES): + for i in range(settings.MAX_RETRIES): try: task.run() break except: - if i < app_settings.MAX_RETRIES - 1: + if i < settings.MAX_RETRIES - 1: pass else: raise From b008f9ff8cda08e5c7c72a5c6717f5f1218ba7b1 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:19:16 +1200 Subject: [PATCH 017/109] refactor: reducing duplicatd code --- manage.sh | 5 +++++ run_test_app.sh | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100755 manage.sh diff --git a/manage.sh b/manage.sh new file mode 100755 index 0000000..d1dfc77 --- /dev/null +++ b/manage.sh @@ -0,0 +1,5 @@ +#!/bin/sh +APP=$1 +shift +source venv/bin/activate +django-admin $@ --pythonpath=. --settings=$APP.settings diff --git a/run_test_app.sh b/run_test_app.sh index a512e03..d79b3e8 100755 --- a/run_test_app.sh +++ b/run_test_app.sh @@ -1,4 +1,3 @@ #!/bin/sh -. venv/bin/activate -django-admin migrate --pythonpath=. --settings=test_app.settings -django-admin runserver --pythonpath=. --settings=test_app.settings +./manage.sh test_app migrate +./manage.sh test_app runserver From c1a1d0a9e1c6b83b385e66d3a99dc7cdd3d711f5 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:21:59 +1200 Subject: [PATCH 018/109] remove unused scripts --- django_leek/service-start.py | 12 ------------ django_leek/shell.py | 18 ------------------ 2 files changed, 30 deletions(-) delete mode 100644 django_leek/service-start.py delete mode 100644 django_leek/shell.py diff --git a/django_leek/service-start.py b/django_leek/service-start.py deleted file mode 100644 index 4122896..0000000 --- a/django_leek/service-start.py +++ /dev/null @@ -1,12 +0,0 @@ -# run from shell with python -m and the full python path -# to allow relative imports -from .settings import TASKS_PORT -from . import worker_manager -from .server import TaskSocketServerThread -import time - -worker_manager.start() -server_thread = TaskSocketServerThread('localhost', TASKS_PORT) -time.sleep(5) -socket_server = server_thread.socket_server() -socket_server.serve_forever() diff --git a/django_leek/shell.py b/django_leek/shell.py deleted file mode 100644 index 7702f51..0000000 --- a/django_leek/shell.py +++ /dev/null @@ -1,18 +0,0 @@ -import socket -import sys -from settings import TASKS_HOST, TASKS_PORT - - -def send_data(): - data = sys.argv[1] - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((TASKS_HOST, TASKS_PORT)) - sock.send(data) - received = sock.recv(1024) - sock.close() - print "Sent: %s" % data - print "Received: %s" % received - - -if __name__ == "__main__": - send_data() From a47e5b19d72a9248b83c08679bb763769c648167 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:22:50 +1200 Subject: [PATCH 019/109] 2to3: print + local import --- django_leek/service-stop-worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_leek/service-stop-worker.py b/django_leek/service-stop-worker.py index 7a317e0..2fc725e 100644 --- a/django_leek/service-stop-worker.py +++ b/django_leek/service-stop-worker.py @@ -1,5 +1,5 @@ import socket -from settings import TASKS_HOST, TASKS_PORT +from .settings import TASKS_HOST, TASKS_PORT def stop_server(): @@ -8,8 +8,8 @@ def stop_server(): sock.send("stop") received = sock.recv(1024) sock.close() - print "Sent: %s" % "stop" - print "Received: %s" % received + print("Sent: %s" % "stop") + print("Received: %s" % received) if __name__ == "__main__": From a21d27dbc7182a30813604dc1948d90e3bb45006 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:28:09 +1200 Subject: [PATCH 020/109] fixing linting --- django_leek/run_failed_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_leek/run_failed_tasks.py b/django_leek/run_failed_tasks.py index 4530f8e..b7a3ef6 100644 --- a/django_leek/run_failed_tasks.py +++ b/django_leek/run_failed_tasks.py @@ -11,7 +11,7 @@ # edit to the correct mysite.tasks_queue path -from mysite.tasks_queue.models import FailedTasks,QueuedTasks +from .models import FailedTasks, QueuedTasks Lfailed_tasks_id = FailedTasks.objects.values_list("task_id", flat=True) From 9849e0e60ff6f627654558d042647476c139cc16 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:28:17 +1200 Subject: [PATCH 021/109] ex: pylintrc --- .pylintrc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..e33918f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,3 @@ + [MASTER] + errors-only=yes + load-plugins=pylint_django From 3e35abaa027d6877e5b4f975ff463a5fb8591e44 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:28:40 +1200 Subject: [PATCH 022/109] ex: linting passes thanks to `pylint-django` --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index c5c5dd6..7c15def 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ django==1.11 pylint +pylint-django From c6e140e95a065275e572ae087c3bb7fd6723458e Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 11:28:53 +1200 Subject: [PATCH 023/109] ex: travis build --- .travis.yml | 8 ++++++++ README.md | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8ceb0a2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "3.5" + - "3.6" +cache: pip +script: + - pylint django_leek + - ./manage.sh django_leek test diff --git a/README.md b/README.md index be9f297..b912fbc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Django Queue +# Django Leek +[![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) **A simple async tasks queue via a django app and SocketServer, zero configs.** From 836cb5de352db2b2ae14edc18935859abdc0c1a3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 14:00:31 +1200 Subject: [PATCH 024/109] freshen up README --- README.md | 221 +++++++++++------------------------------------------- logo.svg | 1 + 2 files changed, 44 insertions(+), 178 deletions(-) create mode 100644 logo.svg diff --git a/README.md b/README.md index b912fbc..80322a3 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,55 @@ # Django Leek +![Logo](logo.svg) + [![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) -**A simple async tasks queue via a django app and SocketServer, zero configs.** +The _simple_ and _slick_ way to run async tasks in Django. -[Why?](#why) -[Overview](#overview) -[Install](#install) -[Settings](#settings) -[Run the Tasks Queue Server](#run-the-tasks-queue-server) -[Persistency](#persistency) -[Failed Tasks](#failed-tasks) -[Run the Tasks Queue on Another Server](#run-the-tasks-queue-on-another-server) +* Django-friendly API +* Easy to start and stop +Based on [django-queue](https://github.com/Aviah/django-queue). ## Why? +With a healthy mix of vegetables, such as [Celery](celeryproject.org) and [Carrot](http://www.django-carrot.com/) aleady in the midst, what does `django-leek` bring? -Although Celery is pretty much the standard for a django tasks queue solution, it can be complex to install and config. +The most "lightweight" library so far has "install Redis" as step one. Although, Redis is a fantastic software, sometimes you just want a simple way of offload the webserver and run a task async, such as sending an email. -The common case for a web application queue is to send emails: you don't want the django thread to wait until the SMTP, or email provider API, finishes. But to send emails from a site without a lot of traffic, or to run other similar simple tasks, you don't need Celery. +Here `django-leek` comes to the rescue. Usage and architecture cannot be simpler, and with so few moving parts, it should be very stable, although it's still not battle tested as e.g. Celery. -This queue app is a simple, up and running queueing solution. The more complex distributed queues can wait until the website has a lot of traffic, and the scalability is really required. +With `django-leek` you can get up and running quickly The more complex distributed queues can wait until the website has a lot of traffic, and the scalability is really required. -## Overview +## Getting started +1. Install `django-leek` with pip + ```bash + $ pip install django-leek + ```` -In a nutshell, a python SocketServer runs in the background, and listens to a tcp socket. SocketServer gets the request to run a task from it's socket, puts the task on a Queue. A Worker thread picks tasks from this Queue, and runs the tasks one by one. +2. Add `django-leek` to `INSTALLED_APPS` in your `settings.py` file. -You send a task request to the SocketServer with: +3. Create tables needed - - from mysite.tasks_queue.API import push_task_to_queue - ... - push_task_to_queue(a_callable,*args,**kwargs) - -Sending email might look like: + ```bash + $ manange.py migrate + ``` + +4. Make sure the django-leek server is running. + + ```bash + $ python manage.py runleek + ``` + +5. Go nuts + + ```python + push_task_to_queue(send_mail, to='foobar@example.com') + ``` + +## Technical overview +In a nutshell, a python SocketServer runs in the background, listening on a tcp socket. SocketServer gets the request to run a task from it's socket, puts the task on a Queue. A Worker thread picks tasks from this Queue, and runs the tasks one by one. - push_task_to_queue(send_mail,subject="foo",message="baz",recipient_list=[user.email]) - ### Components 1. Python SocketServer that listens to a tcp socket. @@ -54,34 +66,17 @@ The workflow that runs an async task: 5. The `Worker` thread runs the task. ### Can this queue scale to production? - -Depends on the traffic: SocketServer is simple, but solid, and as the site gets more traffic, it's possible to move the django-queue server to another machine, separate database etc. At some point, probably, it's better to pick Celery. Until then, django-queue is a simple, solid, and no-hustle solution. - - - -## Install - -1. Add the `tasks_queue` app to your django project -2. Replace `mysite` in the `tasks_queue/worker.py` with the full path to the `tasks_queue` in your project. It should look like the path in the project's manage.py. -2. Add the tasks_queue app to `INSTALLED_APPS` -3. Migrate: - - $ manange.py migrate - -4. The tasks_queue app has an API module, with a `push_task_to_queue` function. Use this function to send callables with args and kwargs to the queue, for the async run. - +Depends on the traffic: SocketServer is simple, but solid, and as the site gets more traffic, it's possible to move the django-queue server to another machine, separate database etc. At some point, probably, it's better to pick Celery. Until then, django-leek is a simple, solid, and no-hustle solution. ## Settings - To change the default django-queue settings, add a `TASKS_QUEUE` dictionary to your project main `settings.py` file. This is the dictionary and the defaults: - TASKS_QUEUE = { - "MAX_RETRIES":3, - "TASKS_HOST":"localhost", - "TASKS_PORT":8002} + "MAX_RETRIES": 3, + "TASKS_HOST": "localhost", + "TASKS_PORT": 8002} **MAX_RETRIES** The number of times the Worker thread will try to run a task before skipping it. The default is 3. @@ -91,77 +86,8 @@ The host that runs the SocketServer. the default is 'localhost'. **TASKS_PORT** The port that SocketServer listens to. The default is 8002. - - - -## Run the Tasks Queue Server - - -###Start the Server -From shell: - - $ python -m mysite.tasks_queue.service-start & - -Provide the full path, without the .py extention. - - -*Note: The tasks queue uses relative imports, and thus should run as a package. If you want to run it with the common `python service-start.py &`, -then edit the imports of the `tasks_queue` files, and convert all the imports to absolute imports.* - - -###Stop the Server - -First stop the worker thread gracefully: - - $ python tasks_queue/service-stop-worker.py - -This will send a stop event to the Worker thread. -Check that the Worker thread stopped: - - $ python tasks_queue/shell.py ping - Sent: ping - Received: (False, 'Worker Off') - -Now you can safely stop SocketServer: - - $ ps ax | grep tasks_queue - 12345 pts/1 S 7:20 python -m mysite.tasks_queue.service-start - $ sudo kill 12345 - - -###Ping the Server - -From shell: - - $ python tasks_queue/shell.py ping - Sent: ping - Received: (True, "I'm OK") - -### Tasks that are waiting on the Queue - -From shell: - - $ python tasks_queue/shell.py waiting - Sent: waiting - Received: (True, 115) - -115 tasks are waiting on the queue - -### Count total tasks handled to the Queue - -From shell: - - $ python tasks_queue/shell.py handled - Sent: handled - Received: (True, 862) - -Total of 862 tasks were handled to the Queue from the moment the thread started - - -*Note: If you use the tasks server commands a lot, add shell aliases for these commands* - -## Persistency +## Persistence ### Tasks saved in the database @@ -169,7 +95,6 @@ Total of 862 tasks were handled to the Queue from the moment the thread started The model saves every tasks pushed to the queue. The task is pickled as a `tasks_queue.tasks.Task` object, which is a simple class with a `callable`,`args` and `kwargs` attributes, and one method: `run()` - **SuccessTasks** The Worker thread saves to this model the `task_id` of every task that was carried out successfuly. **task_id** is the task's `QueuedTasks` id. @@ -177,7 +102,6 @@ The Worker thread saves to this model the `task_id` of every task that was carri After the Worker tries to run a task several times according to `MAX_RETRIES`, and the task still fails, the Worker saves it to this model. The failed taks is saved by the `task_id`, with the exception message. Only the exception from the last run is saved. - ### Purge Tasks According to your project needs, you can purge tasks that the Worker completed successfuly. @@ -192,7 +116,6 @@ The SQL to delete these tasks: In a similar way, delete the failed tasks. You can run a cron script, or other script, to purge the tasks. - ## Failed Tasks ### Retry failed tasks with a script @@ -204,67 +127,9 @@ To re-try failed tasks, after they are saved to the database, you can run this s $ python tasks_queue/run_failed_tasks.py *Note: The path is provided in the script with `mysite`. Edit this entry with the full path to the tasks_queue in your project, similar to the path provided in the project's manage.py* - -### Connections -If most of the tasks require a specific connection, such as SMTP or a database, you can edit the Worker class and add a ping or other check for this connection **before*8 the tasks runs. If the connection is not avaialable, just try to re-connect. - -Otherwise the Worker will just run and fail a lot of tasks. - -## Run the Tasks Queue on Another Server - -The same `tasks_queue` app can run from another server, and provide a seprate server queue for the async tasks. - -Here is a simple way to do it: - -1. The queue server should be similar to the main django server, just without a webserver. -2. Deploy your django code to these two remotes: the main with the web-server, and the queue server -3. Open firewalls ports between the main django server, and the queue server, for the tasks_queue TASKS_PORT, and between the tasks_queue server and the databse server, for the database ports. -5. On the django main server, set TASKS_HOST to the tasks_queue server. -That's it! Now run the server in tasks_queue server with `service-start`, and the main django server will put the tasks to this tasks_queue server. -*Note: "main django server" can be more than one server that run django, and push message to the django queue server* - - - -Support this project with my affiliate link| --------------------------------------------| -https://www.linode.com/?r=cc1175deb6f3ad2f2cd6285f8f82cefe1f0b3f46| - - - - - - - - - - +## Authors +Aviah and Samuel Carlsson - - - - - - - - - - - - - - - - - - - - - - - - - - - +See [contributors]( https://github.com/Volumental/django-leek/graphs/contributors) for full list. diff --git a/logo.svg b/logo.svg new file mode 100644 index 0000000..f68d682 --- /dev/null +++ b/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file From 43daa6b8409bebcf92ba4147f31e7637db63e92b Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 15:04:21 +1200 Subject: [PATCH 025/109] decorator api --- django_leek/API.py | 21 ++++++++++++++++----- test_app/app.py | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/django_leek/API.py b/django_leek/API.py index 7803e64..24b2a1a 100644 --- a/django_leek/API.py +++ b/django_leek/API.py @@ -1,16 +1,27 @@ import socket +from functools import wraps from .task import Task from . import helpers from .settings import TASKS_HOST,TASKS_PORT -def push_task_to_queue(a_callable,*args,**kwargs): +class Leek(object): + def task(self, f): + @wraps(f) + def _offload(*args, **kwargs): + return push_task_to_queue(f, *args, **kwargs) + f.offload = _offload + return f + + +def push_task_to_queue(a_callable, *args, **kwargs): + """Original API""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - new_task = Task(a_callable,*args,**kwargs) - new_task = helpers.save_task_to_db(new_task) #returns with db_id + new_task = Task(a_callable, *args, **kwargs) + new_task = helpers.save_task_to_db(new_task) # returns with db_id sock.connect((TASKS_HOST, TASKS_PORT)) sock.send(helpers.serielize(new_task)) received = sock.recv(1024) sock.close() - - return received \ No newline at end of file + + return received diff --git a/test_app/app.py b/test_app/app.py index 7f21609..ec3252a 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -2,15 +2,24 @@ from django.http import HttpResponse from django.shortcuts import render -from django_leek.API import push_task_to_queue +from django_leek.API import Leek, push_task_to_queue -def task(): - print('Executed on queue!') +leek = Leek() + +@leek.task +def hello(to): + print('Hello {}!'.format(to)) def index(request): if 'queue' in request.GET: - push_task_to_queue(task) + # Run sync + hello(to='sync') + + # Run async + hello.offload(to='new') + + push_task_to_queue(hello, to='old') return render(request, 'index.html', {'message': '✓ task queued'}) return render(request, 'index.html') From debd24768a9c34e889ca88cd618350d3f95997e6 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 15:06:09 +1200 Subject: [PATCH 026/109] refactor: module in lowercase --- django_leek/{API.py => api.py} | 0 test_app/app.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename django_leek/{API.py => api.py} (100%) diff --git a/django_leek/API.py b/django_leek/api.py similarity index 100% rename from django_leek/API.py rename to django_leek/api.py diff --git a/test_app/app.py b/test_app/app.py index ec3252a..8117207 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -2,7 +2,7 @@ from django.http import HttpResponse from django.shortcuts import render -from django_leek.API import Leek, push_task_to_queue +from django_leek.api import Leek, push_task_to_queue leek = Leek() From f3f22d68d8004652f5fab999b93774e2af1caca1 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 15:26:43 +1200 Subject: [PATCH 027/109] dist: depend on django --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e5348e7..ea6c49e 100644 --- a/setup.py +++ b/setup.py @@ -13,14 +13,15 @@ name='django-leek', version='0.1', packages=['django_leek'], + install_requires = ['django>=1.11'], include_package_data=True, license='MIT License', description='A simple Django app to offload tasks from main web server', long_description=README, long_description_content_type='text/markdown', url='https://github.com/Volumental/django-leek', - author='Samuel Carlsson', - author_email='samuel.carlsson@volumental.com', + author='Volumental', + author_email='maintainer@volumental.com', classifiers=[ 'Environment :: Web Environment', 'Framework :: Django', From d50d653db8b94ddc0ab8ec666fda65076c600777 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 15:28:33 +1200 Subject: [PATCH 028/109] v0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea6c49e..47a12e0 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.1', + version='0.2', packages=['django_leek'], install_requires = ['django>=1.11'], include_package_data=True, From c1b56d200576b6a9166c0e4554ee797c5628afc8 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 16:00:34 +1200 Subject: [PATCH 029/109] fixup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47a12e0..50e6e91 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name='django-leek', version='0.2', - packages=['django_leek'], + packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, license='MIT License', From 89a69525a47445fa6dbdf07ac1a70fe315ce8ddd Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 16:13:15 +1200 Subject: [PATCH 030/109] v0.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50e6e91..bf50f7c 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.2', + version='0.3', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From ab9f7534d761a72b33bb0a5345780ea4b5b781e4 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 16:27:07 +1200 Subject: [PATCH 031/109] ex: code coverage --- .coveragerc | 3 +++ .travis.yml | 2 +- requirements.txt | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..1f8a301 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +source=django_leek +omit=venv/*, test_app/* diff --git a/.travis.yml b/.travis.yml index 8ceb0a2..da60494 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,4 @@ python: cache: pip script: - pylint django_leek - - ./manage.sh django_leek test + - coverage run $(which django-admin) test --pythonpath=. --settings=django_leek.settings diff --git a/requirements.txt b/requirements.txt index 7c15def..742ec2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ django==1.11 pylint pylint-django +coverage From 97c1b8b1960e45a086f41f4df6803703609084be Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 8 May 2018 16:27:49 +1200 Subject: [PATCH 032/109] ex: publish to codecov --- .travis.yml | 1 + README.md | 2 +- requirements.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index da60494..21ebd81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,3 +6,4 @@ cache: pip script: - pylint django_leek - coverage run $(which django-admin) test --pythonpath=. --settings=django_leek.settings + - codecov diff --git a/README.md b/README.md index 80322a3..bbe2cbc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Logo](logo.svg) [![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) - +[![Code coverage](https://codecov.io/gh/Volumental/django-leek/branch/master/graph/badge.svg)](https://codecov.io/gh/Volumental/django-leek) The _simple_ and _slick_ way to run async tasks in Django. * Django-friendly API diff --git a/requirements.txt b/requirements.txt index 742ec2f..7779da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ django==1.11 pylint pylint-django coverage +codecov From 05be29bb7b5e9df6f1f221716f8de14714c52cef Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 09:47:45 +1200 Subject: [PATCH 033/109] Using model in background worker --- test_app/app.py | 9 +++++++++ test_app/migrations/0001_initial.py | 23 +++++++++++++++++++++++ test_app/migrations/__init__.py | 0 test_app/settings.py | 1 + 4 files changed, 33 insertions(+) create mode 100644 test_app/migrations/0001_initial.py create mode 100644 test_app/migrations/__init__.py diff --git a/test_app/app.py b/test_app/app.py index 8117207..3a66e98 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -1,13 +1,22 @@ from django.conf.urls import url from django.http import HttpResponse from django.shortcuts import render +from django.db import models from django_leek.api import Leek, push_task_to_queue leek = Leek() + +class Person(models.Model): + name = models.CharField(max_length=30) + + @leek.task def hello(to): + person = Person.objects.create(name="to") + person.save() + print('Hello {}!'.format(to)) diff --git a/test_app/migrations/0001_initial.py b/test_app/migrations/0001_initial.py new file mode 100644 index 0000000..7b37bfe --- /dev/null +++ b/test_app/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-05-08 16:46 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Person', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=30)), + ], + ), + ] diff --git a/test_app/migrations/__init__.py b/test_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_app/settings.py b/test_app/settings.py index 7cc4029..f3176aa 100644 --- a/test_app/settings.py +++ b/test_app/settings.py @@ -5,6 +5,7 @@ DEBUG=True INSTALLED_APPS = [ + 'test_app', 'django_leek' ] From 8d01a96cc26280bac38f87835076fc2c4071f7d0 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 15:29:00 +1200 Subject: [PATCH 034/109] hacking around --- django_leek/api.py | 4 +-- django_leek/management/commands/leek.py | 32 ++++++++++++++++++++++ django_leek/management/commands/runleek.py | 19 ------------- django_leek/server.py | 15 ---------- django_leek/settings.py | 18 +++--------- 5 files changed, 38 insertions(+), 50 deletions(-) create mode 100644 django_leek/management/commands/leek.py delete mode 100644 django_leek/management/commands/runleek.py diff --git a/django_leek/api.py b/django_leek/api.py index 24b2a1a..dd03e03 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -2,7 +2,7 @@ from functools import wraps from .task import Task from . import helpers -from .settings import TASKS_HOST,TASKS_PORT +from .settings import HOST, PORT class Leek(object): @@ -19,7 +19,7 @@ def push_task_to_queue(a_callable, *args, **kwargs): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_task = Task(a_callable, *args, **kwargs) new_task = helpers.save_task_to_db(new_task) # returns with db_id - sock.connect((TASKS_HOST, TASKS_PORT)) + sock.connect((HOST, PORT)) sock.send(helpers.serielize(new_task)) received = sock.recv(1024) sock.close() diff --git a/django_leek/management/commands/leek.py b/django_leek/management/commands/leek.py new file mode 100644 index 0000000..d46fd74 --- /dev/null +++ b/django_leek/management/commands/leek.py @@ -0,0 +1,32 @@ +import socketserver + +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +from django_leek import worker_manager +from django_leek.server import TaskSocketServer + + +def _endpoint(endpoint): + host, port = endpoint.split(':') + return host, int(port) + + +class Command(BaseCommand): + help = 'Starts leek worker server' + + #def add_arguments(self, parser): + # parser.add_argument('poll_id', nargs='+', type=int) + + def handle(self, *args, **options): + try: + worker_manager.start() + cfg = getattr(settings, 'LEEK', {}) + host, port = _endpoint(cfg.get('bind', 'localhost:8002')) + + print('Listening on {port}'.format(port=port)) + server = socketserver.TCPServer((host, port), TaskSocketServer) + server.serve_forever() + except KeyboardInterrupt: + pass + print('exiting') diff --git a/django_leek/management/commands/runleek.py b/django_leek/management/commands/runleek.py deleted file mode 100644 index d98bc86..0000000 --- a/django_leek/management/commands/runleek.py +++ /dev/null @@ -1,19 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - - -class Command(BaseCommand): - help = 'Starts leek worker server' - - #def add_arguments(self, parser): - # parser.add_argument('poll_id', nargs='+', type=int) - - def handle(self, *args, **options): - from django_leek import worker_manager - from django_leek.server import TaskSocketServerThread - import time - - worker_manager.start() - server_thread = TaskSocketServerThread('localhost', 8002) - time.sleep(5) - socket_server = server_thread.socket_server() - socket_server.serve_forever() diff --git a/django_leek/server.py b/django_leek/server.py index 92dfc42..fea7b06 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -40,18 +40,3 @@ def handle(self): self.request.send(str(response)) except Exception as e: self.request.send("SocketServer Response: {}".format(e).encode()) - - -class TaskSocketServerThread(threading.Thread): - def __init__(self,host,port): - threading.Thread.__init__(self, name='tasks-socket-server') - self.host = host - self.port = port - self.setDaemon(1) - self.start() - - def socket_server(self): - return self.server - - def run(self): - self.server = socketserver.TCPServer((self.host, self.port), TaskSocketServer) diff --git a/django_leek/settings.py b/django_leek/settings.py index 7ad2567..ba04ed3 100644 --- a/django_leek/settings.py +++ b/django_leek/settings.py @@ -3,18 +3,8 @@ SECRET_KEY="just to make tests run" -D = { - "MAX_RETRIES": 3, - "TASKS_HOST": "localhost", - "TASKS_PORT": 8002 -} - - -if hasattr(settings,"TASKS_QUEUE"): - for key,value in getattr(settings, "TASKS_QUEUE"): - D[key] = value - +cfg = getattr(settings, "LEEK", {}) -MAX_RETRIES = D["MAX_RETRIES"] -TASKS_HOST = D['TASKS_HOST'] -TASKS_PORT = D['TASKS_PORT'] +MAX_RETRIES = cfg.get('max_retries', 3) +HOST = cfg.get('host', "localhost") +PORT = int(cfg.get('port', "8002")) From 7ccb3926c5c80f7c9bce5cb0fc05a0588c6f2600 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 15:38:28 +1200 Subject: [PATCH 035/109] hot up readme --- README.md | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index bbe2cbc..04d8554 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) [![Code coverage](https://codecov.io/gh/Volumental/django-leek/branch/master/graph/badge.svg)](https://codecov.io/gh/Volumental/django-leek) + The _simple_ and _slick_ way to run async tasks in Django. * Django-friendly API @@ -69,27 +70,30 @@ The workflow that runs an async task: Depends on the traffic: SocketServer is simple, but solid, and as the site gets more traffic, it's possible to move the django-queue server to another machine, separate database etc. At some point, probably, it's better to pick Celery. Until then, django-leek is a simple, solid, and no-hustle solution. ## Settings -To change the default django-queue settings, add a `TASKS_QUEUE` dictionary to your project main `settings.py` file. +To change the default django-queue settings, add a `LEEK` dictionary to your project main `settings.py` file. This is the dictionary and the defaults: - TASKS_QUEUE = { - "MAX_RETRIES": 3, - "TASKS_HOST": "localhost", - "TASKS_PORT": 8002} + LEEK = { + 'max_retries': 3, + 'bind': "localhost:8002", + 'host': "localhost", + 'port': 8002} -**MAX_RETRIES** -The number of times the Worker thread will try to run a task before skipping it. The default is 3. +**`max_retries`** +The number of times the Worker thread will try to run a task before skipping it. -**TASKS_HOST** -The host that runs the SocketServer. the default is 'localhost'. +**`bind`** +The leek server will bind here. -**TASKS_PORT** -The port that SocketServer listens to. The default is 8002. +**`host`** +The django server will connect to this host when notifying leek of jobs. -## Persistence +**`port`** +The django server will connect to this port when notifying leek of jobs. -### Tasks saved in the database +## Persistence +The following models are used. **QueuedTasks** The model saves every tasks pushed to the queue. @@ -116,18 +120,6 @@ The SQL to delete these tasks: In a similar way, delete the failed tasks. You can run a cron script, or other script, to purge the tasks. -## Failed Tasks - -### Retry failed tasks with a script - -When the Worker fails to run the task `MAX_RETRIES` times, it saves the **task_id** and the exception message to the `FailedTasks` model. - -To re-try failed tasks, after they are saved to the database, you can run this script, from shell: - - $ python tasks_queue/run_failed_tasks.py - -*Note: The path is provided in the script with `mysite`. Edit this entry with the full path to the tasks_queue in your project, similar to the path provided in the project's manage.py* - ## Authors Aviah and Samuel Carlsson From cfb7d32973cafa750e1b3eecd0ff56ac25a556f4 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 15:38:48 +1200 Subject: [PATCH 036/109] version 0.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf50f7c..b44910b 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.3', + version='0.4', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 4bf0f82b6409f6a96997dc3de3978431dfa34a7d Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 15:45:34 +1200 Subject: [PATCH 037/109] v0.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b44910b..d00744c 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.4', + version='0.5', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From a08e9ff70138edadccac1f1c7bacd53aa3ff0acb Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 9 May 2018 15:47:43 +1200 Subject: [PATCH 038/109] fix build --- django_leek/service-stop-worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_leek/service-stop-worker.py b/django_leek/service-stop-worker.py index 2fc725e..ccf5fdd 100644 --- a/django_leek/service-stop-worker.py +++ b/django_leek/service-stop-worker.py @@ -1,10 +1,10 @@ import socket -from .settings import TASKS_HOST, TASKS_PORT +from .settings import HOST, PORT def stop_server(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((TASKS_HOST, TASKS_PORT)) + sock.connect((HOST, PORT)) sock.send("stop") received = sock.recv(1024) sock.close() From cb01c70db8de8066c4bd6900b7fa280ba48d7652 Mon Sep 17 00:00:00 2001 From: Miroslav Kobetski Date: Tue, 15 May 2018 14:07:05 +0200 Subject: [PATCH 039/109] send wants a bytes object --- django_leek/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_leek/server.py b/django_leek/server.py index fea7b06..72b8078 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -37,6 +37,6 @@ def handle(self): response = (False, "TaskServer Put: {}".format(e).encode(),) try: - self.request.send(str(response)) + self.request.send(str(response).encode('utf-8')) except Exception as e: self.request.send("SocketServer Response: {}".format(e).encode()) From 83f70be65f07ffd5e85fa9c1e6607e495d159df1 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 13:44:31 +1200 Subject: [PATCH 040/109] doc: updating documentation --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 04d8554..ad4a6cf 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,31 @@ With `django-leek` you can get up and running quickly The more complex distribut 5. Go nuts + ```python + leek = Leek() + @leek.task + def send_mail(to): + do_what_ever() + + send_mail.offload(to='foobar@example.com') + ``` + + You can also use the "old" as found in `django-queue` ```python push_task_to_queue(send_mail, to='foobar@example.com') ``` +6. It's easy to unit test code that in production offloads work to the Leek server. + + ```python + def _invoke(a_callable, *args, **kwargs): ++ a_callable(*args, **kwargs) + @patch('django_leek.api.push_task_to_queue', _invoke) + def test_mytest(): + send_mail.offload(to='sync@leek.com') # now runs synchronously, like a normal function + ``` + + ## Technical overview In a nutshell, a python SocketServer runs in the background, listening on a tcp socket. SocketServer gets the request to run a task from it's socket, puts the task on a Queue. A Worker thread picks tasks from this Queue, and runs the tasks one by one. From 882ad6b6289fe9275523d446962c560b05f869a3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 13:58:40 +1200 Subject: [PATCH 041/109] ex: unit test for starting server --- django_leek/settings.py | 7 ++++++- django_leek/tests.py | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/django_leek/settings.py b/django_leek/settings.py index ba04ed3..08746d6 100644 --- a/django_leek/settings.py +++ b/django_leek/settings.py @@ -1,7 +1,12 @@ +import sys from django.conf import settings -SECRET_KEY="just to make tests run" +if 'test' in sys.argv: + SECRET_KEY="just to make tests run" + DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3'}} + INSTALLED_APPS = ['django_leek'] + cfg = getattr(settings, "LEEK", {}) diff --git a/django_leek/tests.py b/django_leek/tests.py index 7ce503c..cf1a2f1 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,3 +1,11 @@ +from unittest.mock import patch +import socketserver + from django.test import TestCase +from django.core.management import call_command -# Create your tests here. +class LeekCommandTestCase(TestCase): + @patch.object(socketserver.TCPServer, 'serve_forever') + def test_leek(self, serve_forever): + call_command('leek') + serve_forever.assert_called() From 428feee3167c683acb95fee4e645d06cf47b518e Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 14:01:44 +1200 Subject: [PATCH 042/109] ex: unit test keyboard interrupt --- django_leek/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_leek/tests.py b/django_leek/tests.py index cf1a2f1..22df8a3 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -4,8 +4,12 @@ from django.test import TestCase from django.core.management import call_command +@patch.object(socketserver.TCPServer, 'serve_forever') class LeekCommandTestCase(TestCase): - @patch.object(socketserver.TCPServer, 'serve_forever') def test_leek(self, serve_forever): call_command('leek') serve_forever.assert_called() + + def test_keyboard_interrupt(self, serve_forever): + serve_forever.side_effect = KeyboardInterrupt + call_command('leek') From 49a3c24acfc9257d024a7db3fc6eac2d15da7246 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 15:16:12 +1200 Subject: [PATCH 043/109] ex: test server --- django_leek/settings.py | 2 +- django_leek/tests.py | 45 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/django_leek/settings.py b/django_leek/settings.py index 08746d6..a7d2a23 100644 --- a/django_leek/settings.py +++ b/django_leek/settings.py @@ -4,7 +4,7 @@ if 'test' in sys.argv: SECRET_KEY="just to make tests run" - DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3'}} + DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} INSTALLED_APPS = ['django_leek'] diff --git a/django_leek/tests.py b/django_leek/tests.py index 22df8a3..aa6bf2e 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,9 +1,16 @@ -from unittest.mock import patch +import base64 +import pickle +from unittest.mock import patch, MagicMock import socketserver from django.test import TestCase from django.core.management import call_command +from django_leek.server import TaskSocketServer +from django_leek.task import Task +from django_leek import helpers + + @patch.object(socketserver.TCPServer, 'serve_forever') class LeekCommandTestCase(TestCase): def test_leek(self, serve_forever): @@ -13,3 +20,39 @@ def test_leek(self, serve_forever): def test_keyboard_interrupt(self, serve_forever): serve_forever.side_effect = KeyboardInterrupt call_command('leek') + +def f(): + pass + +class TestServer(TestCase): + def setUp(self): + self.request = MagicMock() + + def _request(self, data): + if isinstance(data, Exception): + self.request.recv.side_effect = data + else: + self.request.recv.return_value = data + + def _response(self): + return b''.join(call[0][0] for call in self.request.send.call_args_list) + + def act(self): + TaskSocketServer(self.request, 'client adress', 'server') + + def test_ping(self): + self._request('ping') + self.act() + self.assertEqual(self._response(), b'(True, b"I\'m OK")') + + # This apparently crashes the server + #def test_recv_error(self): + # self._request(OSError('Nuclear Winter')) + # self.act() + # self.assertEqual(self._response(), b'(True, b"I\'m OK")') + + def test_task(self): + task = helpers.save_task_to_db(Task(f)) + self._request(base64.b64encode(pickle.dumps(task))) + self.act() + self.assertEqual(self._response(), b"(True, 'sent')") From b23233eca77a4b6fb5c491e74e52cf60493bd5b7 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 15:24:02 +1200 Subject: [PATCH 044/109] ex: supporting 3.5 --- django_leek/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_leek/tests.py b/django_leek/tests.py index aa6bf2e..a59c32c 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -15,7 +15,7 @@ class LeekCommandTestCase(TestCase): def test_leek(self, serve_forever): call_command('leek') - serve_forever.assert_called() + serve_forever.assert_called_with() def test_keyboard_interrupt(self, serve_forever): serve_forever.side_effect = KeyboardInterrupt From 4b76dea5e30c2d06f0b00f3d6471603f9739453f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 16 May 2018 15:37:14 +1200 Subject: [PATCH 045/109] ex: coveralls switching to coveralls from codecov --- .travis.yml | 2 +- README.md | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21ebd81..b8c9c62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ cache: pip script: - pylint django_leek - coverage run $(which django-admin) test --pythonpath=. --settings=django_leek.settings - - codecov + - coveralls diff --git a/README.md b/README.md index ad4a6cf..880e29b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Logo](logo.svg) [![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) -[![Code coverage](https://codecov.io/gh/Volumental/django-leek/branch/master/graph/badge.svg)](https://codecov.io/gh/Volumental/django-leek) +[![Coverage Status](https://coveralls.io/repos/github/Volumental/django-leek/badge.svg?branch=master)](https://coveralls.io/github/Volumental/django-leek?branch=master) The _simple_ and _slick_ way to run async tasks in Django. diff --git a/requirements.txt b/requirements.txt index 7779da4..236967f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ django==1.11 pylint pylint-django coverage -codecov +coveralls From 1f7df04575f9c41a9a7a18a45c6211d96b04d0be Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:09:09 +0200 Subject: [PATCH 046/109] leek would spinlock using 100% CPU. Huge drainer on the cloud bill. --- django_leek/worker.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index f64b5ec..f449084 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -79,14 +79,13 @@ def run(self): django.setup() self.logger.info('Worker Starts') while not self._stopevent.isSet(): - if not self.worker_queue.empty(): - try: - task = self.worker_queue.get() - self.run_task(task) - except Exception as e: - helpers.save_task_failed(task,e) - else: - helpers.save_task_success(task) + try: + task = self.worker_queue.get() + self.run_task(task) + except Exception as e: + helpers.save_task_failed(task,e) + else: + helpers.save_task_success(task) self.worker_queue = None self.logger.warn('Worker stopped, %s tasks handled'%self.tasks_counter) From 6781bbf53d4838e41f001f3d485c69e543a1fb45 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:17:17 +0200 Subject: [PATCH 047/109] ex: manage.sh on linux manage.sh did not work on linux. Now it does. --- manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.sh b/manage.sh index d1dfc77..c84ea8f 100755 --- a/manage.sh +++ b/manage.sh @@ -1,5 +1,5 @@ #!/bin/sh APP=$1 shift -source venv/bin/activate +. venv/bin/activate django-admin $@ --pythonpath=. --settings=$APP.settings From 9b446a8b27312d83ae4ab878f3feb0ff28ea7f6c Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:19:14 +0200 Subject: [PATCH 048/109] ex: going coverage-shield --- .travis.yml | 2 +- README.md | 2 +- requirements.txt | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8c9c62..9d146b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ cache: pip script: - pylint django_leek - coverage run $(which django-admin) test --pythonpath=. --settings=django_leek.settings - - coveralls + - python -m coverage_shield diff --git a/README.md b/README.md index 880e29b..f647c5d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Logo](logo.svg) [![Build Status](https://travis-ci.com/Volumental/django-leek.svg?branch=master)](https://travis-ci.com/Volumental/django-leek) -[![Coverage Status](https://coveralls.io/repos/github/Volumental/django-leek/badge.svg?branch=master)](https://coveralls.io/github/Volumental/django-leek?branch=master) +![Code Coverage](https://badges-io.now.sh/badge/Volumental/django-leek/coverage.svg) The _simple_ and _slick_ way to run async tasks in Django. diff --git a/requirements.txt b/requirements.txt index 236967f..f9592bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ django==1.11 + +# for development pylint pylint-django coverage -coveralls +coverage-shield From 1d6293129c5d61b8861fab32599f1d10992105f1 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:20:31 +0200 Subject: [PATCH 049/109] uping version --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d00744c..4a1fcd0 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.5', + version='0.6', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, @@ -25,9 +25,8 @@ classifiers=[ 'Environment :: Web Environment', 'Framework :: Django', - 'Framework :: Django :: 1.9', # replace "X.Y" as appropriate + 'Framework :: Django :: 1.11', 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', # example license 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3.5', From 8ccf93909e368c5b62c3eabd4e1f88185b69120d Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:25:07 +0200 Subject: [PATCH 050/109] refactor: remiving unused code --- django_leek/worker.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index f449084..160037d 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -6,9 +6,6 @@ import queue -# environ settings variable, should be the same as in manage.py -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") -os.environ["DJANGO_SETTINGS_MODULE"] = "mysite.settings" import django @@ -50,6 +47,17 @@ def run_task(self, task): else: raise + def run_task(self, task): + for i in range(settings.MAX_RETRIES): + try: + task.run() + break + except: + if i < settings.MAX_RETRIES - 1: + pass + else: + raise + def stop_thread(self, timeout=None): """ Stop the thread and wait for it to end. """ if self.worker_queue != None: From 120f240019ac723974aec89735290219def9d54f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:25:36 +0200 Subject: [PATCH 051/109] refactor: removing duplicate function --- django_leek/worker.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index 160037d..f791d3d 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -47,17 +47,6 @@ def run_task(self, task): else: raise - def run_task(self, task): - for i in range(settings.MAX_RETRIES): - try: - task.run() - break - except: - if i < settings.MAX_RETRIES - 1: - pass - else: - raise - def stop_thread(self, timeout=None): """ Stop the thread and wait for it to end. """ if self.worker_queue != None: From ce4d2a25979245fd409a0975ee57e0ea805f9628 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:27:00 +0200 Subject: [PATCH 052/109] removing auto-retry --- README.md | 6 +----- django_leek/settings.py | 1 - django_leek/worker.py | 10 +--------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index f647c5d..26e8cbb 100644 --- a/README.md +++ b/README.md @@ -96,14 +96,10 @@ To change the default django-queue settings, add a `LEEK` dictionary to your pro This is the dictionary and the defaults: LEEK = { - 'max_retries': 3, 'bind': "localhost:8002", 'host': "localhost", 'port': 8002} -**`max_retries`** -The number of times the Worker thread will try to run a task before skipping it. - **`bind`** The leek server will bind here. @@ -124,7 +120,7 @@ The task is pickled as a `tasks_queue.tasks.Task` object, which is a simple clas The Worker thread saves to this model the `task_id` of every task that was carried out successfuly. **task_id** is the task's `QueuedTasks` id. **FailedTasks** -After the Worker tries to run a task several times according to `MAX_RETRIES`, and the task still fails, the Worker saves it to this model. The failed taks is saved by the `task_id`, with the exception message. Only the exception from the last run is saved. +After the Worker tries to run a task and it fails by raising an exception, the Worker saves it to this model. The failed taks is saved by the `task_id`, with the exception message. Only the exception from the last run is saved. ### Purge Tasks diff --git a/django_leek/settings.py b/django_leek/settings.py index a7d2a23..0a4662a 100644 --- a/django_leek/settings.py +++ b/django_leek/settings.py @@ -10,6 +10,5 @@ cfg = getattr(settings, "LEEK", {}) -MAX_RETRIES = cfg.get('max_retries', 3) HOST = cfg.get('host', "localhost") PORT = int(cfg.get('port', "8002")) diff --git a/django_leek/worker.py b/django_leek/worker.py index f791d3d..0f1a23e 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -37,15 +37,7 @@ def put_task_on_queue(self, new_pickled_task): return False, "Worker: %s"%str(e) def run_task(self, task): - for i in range(settings.MAX_RETRIES): - try: - task.run() - break - except: - if i < settings.MAX_RETRIES - 1: - pass - else: - raise + task.run() def stop_thread(self, timeout=None): """ Stop the thread and wait for it to end. """ From 3d1da40379f53c707dade7f1b650c57aa70a8db9 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:29:48 +0200 Subject: [PATCH 053/109] refactor: removing unused stop event --- django_leek/worker.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index 0f1a23e..98d6a94 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -16,7 +16,6 @@ class Worker(threading.Thread): def __init__(self, logger_name=None): threading.Thread.__init__(self, name="django-leek") - self._stopevent = threading.Event() self.setDaemon(1) self.worker_queue = queue.Queue() self.tasks_counter = 0 @@ -40,13 +39,9 @@ def run_task(self, task): task.run() def stop_thread(self, timeout=None): - """ Stop the thread and wait for it to end. """ - if self.worker_queue != None: - self._stopevent.set() - self.logger.warn('Worker stop event set') - return "Stop Set" - else: - return "Worker Off" + """Stop the thread and wait for it to end.""" + self.worker_queue.put(None) + self.join() def ping(self): if self.worker_queue != None: @@ -67,10 +62,14 @@ def run(self): # thread safe locals: L = threading.local(), then L.foo="baz" django.setup() self.logger.info('Worker Starts') - while not self._stopevent.isSet(): + done = False + while not done: try: task = self.worker_queue.get() - self.run_task(task) + if task is None: + done = True + else: + self.run_task(task) except Exception as e: helpers.save_task_failed(task,e) else: From 5c6da2f9392ef706dafc0216bec262a93cdb78d9 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 10:30:24 +0200 Subject: [PATCH 054/109] refactor: simplifying flow --- django_leek/worker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index 98d6a94..ba799d5 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -68,12 +68,12 @@ def run(self): task = self.worker_queue.get() if task is None: done = True - else: - self.run_task(task) + break + + self.run_task(task) + helpers.save_task_success(task) except Exception as e: helpers.save_task_failed(task,e) - else: - helpers.save_task_success(task) - + self.worker_queue = None self.logger.warn('Worker stopped, %s tasks handled'%self.tasks_counter) From 859189f8d084f5f56d4642343ffecd44cc97fa8b Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 11:39:58 +0200 Subject: [PATCH 055/109] bufix: crash on OSError --- django_leek/server.py | 51 +++++++++++++++++++++++-------------------- django_leek/tests.py | 8 +++---- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 72b8078..7d1db30 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,9 +1,13 @@ +import logging import socketserver import threading from . import worker_manager +log = logging.getLogger(__name__) + + Dcommands = { 'ping': worker_manager.ping, 'waiting': worker_manager.waiting, @@ -15,28 +19,27 @@ class TaskSocketServer(socketserver.BaseRequestHandler): def handle(self): try: - data = self.request.recv(5000).strip() #like the pickled task field - except Exception as e: - response = (False, "SocketServer: {}".format(e).encode()) - self.request.send(response) - - if data in Dcommands.keys(): - try: - worker_response = Dcommands[data]() - if worker_response == 'Worker Off': - response = (False, worker_response.encode()) - else: - response = (True, worker_response.encode(),) - except Exception as e: - response = (False, "TaskServer Command: {}".format(e).encode(),) - else: - try: - worker_response = worker_manager.put_task(data) #a tuple - response = worker_response - except Exception as e: - response = (False, "TaskServer Put: {}".format(e).encode(),) + data = self.request.recv(5000).strip() + if data in Dcommands.keys(): + try: + worker_response = Dcommands[data]() + if worker_response == 'Worker Off': + response = (False, worker_response.encode()) + else: + response = (True, worker_response.encode(),) + self.request.send(str(response).encode()) + except Exception as e: + response = (False, "TaskServer Command: {}".format(e).encode(),) + self.request.send(str(response).encode()) + else: + # assume a serialized task + try: + response = worker_manager.put_task(data) + self.request.send(str(response).encode()) + except Exception as e: + response = (False, "TaskServer Put: {}".format(e).encode(),) + self.request.send(str(response).encode()) - try: - self.request.send(str(response).encode('utf-8')) - except Exception as e: - self.request.send("SocketServer Response: {}".format(e).encode()) + except OSError as e: + # in case of network error, just log + log.exception("network error") diff --git a/django_leek/tests.py b/django_leek/tests.py index a59c32c..08b2370 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -45,11 +45,9 @@ def test_ping(self): self.act() self.assertEqual(self._response(), b'(True, b"I\'m OK")') - # This apparently crashes the server - #def test_recv_error(self): - # self._request(OSError('Nuclear Winter')) - # self.act() - # self.assertEqual(self._response(), b'(True, b"I\'m OK")') + def test_recv_error(self): + self._request(OSError('Nuclear Winter')) + self.act() def test_task(self): task = helpers.save_task_to_db(Task(f)) From 36833442f8a4509dd8e08fafdf2f900ce35ff4cb Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 11:41:06 +0200 Subject: [PATCH 056/109] rm: `ping` command --- django_leek/server.py | 6 +----- django_leek/tests.py | 5 ----- django_leek/worker.py | 6 ------ django_leek/worker_manager.py | 4 ---- 4 files changed, 1 insertion(+), 20 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 7d1db30..ba2c2df 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -9,7 +9,6 @@ Dcommands = { - 'ping': worker_manager.ping, 'waiting': worker_manager.waiting, 'handled': worker_manager.hanled, 'stop': worker_manager.stop @@ -23,10 +22,7 @@ def handle(self): if data in Dcommands.keys(): try: worker_response = Dcommands[data]() - if worker_response == 'Worker Off': - response = (False, worker_response.encode()) - else: - response = (True, worker_response.encode(),) + response = (True, worker_response.encode(),) self.request.send(str(response).encode()) except Exception as e: response = (False, "TaskServer Command: {}".format(e).encode(),) diff --git a/django_leek/tests.py b/django_leek/tests.py index 08b2370..289dbe7 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -40,11 +40,6 @@ def _response(self): def act(self): TaskSocketServer(self.request, 'client adress', 'server') - def test_ping(self): - self._request('ping') - self.act() - self.assertEqual(self._response(), b'(True, b"I\'m OK")') - def test_recv_error(self): self._request(OSError('Nuclear Winter')) self.act() diff --git a/django_leek/worker.py b/django_leek/worker.py index ba799d5..c38366c 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -43,12 +43,6 @@ def stop_thread(self, timeout=None): self.worker_queue.put(None) self.join() - def ping(self): - if self.worker_queue != None: - return "I'm OK" - else: - return "Worker Off" - def status_waiting(self): return self.worker_queue.qsize() diff --git a/django_leek/worker_manager.py b/django_leek/worker_manager.py index d5f5f34..11037c8 100644 --- a/django_leek/worker_manager.py +++ b/django_leek/worker_manager.py @@ -14,10 +14,6 @@ def stop(): return worker_thread.stop_thread() -def ping(): - return worker_thread.ping() - - def waiting(): return worker_thread.status_waiting() From a00b3f70717001007747a8ce3c1b2c374042626e Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 1 Jun 2018 11:42:52 +0200 Subject: [PATCH 057/109] better logging --- django_leek/server.py | 4 ++++ django_leek/worker.py | 19 ++++++++++--------- django_leek/worker_manager.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index ba2c2df..6128970 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -20,19 +20,23 @@ def handle(self): try: data = self.request.recv(5000).strip() if data in Dcommands.keys(): + log.info('Got command: "{}"'.format(data)) try: worker_response = Dcommands[data]() response = (True, worker_response.encode(),) self.request.send(str(response).encode()) except Exception as e: + log.exception("command failed") response = (False, "TaskServer Command: {}".format(e).encode(),) self.request.send(str(response).encode()) else: # assume a serialized task + log.info('Got a task') try: response = worker_manager.put_task(data) self.request.send(str(response).encode()) except Exception as e: + log.exception("failed to queue task") response = (False, "TaskServer Put: {}".format(e).encode(),) self.request.send(str(response).encode()) diff --git a/django_leek/worker.py b/django_leek/worker.py index c38366c..4d83f9f 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -13,17 +13,15 @@ from . import helpers +log = logging.getLogger(__name__) + + class Worker(threading.Thread): - def __init__(self, logger_name=None): + def __init__(self): threading.Thread.__init__(self, name="django-leek") self.setDaemon(1) self.worker_queue = queue.Queue() self.tasks_counter = 0 - if logger_name != None: - self.logger = logging.getLogger(logger_name) - else: - self.logger = logging - self.start() def put_task_on_queue(self, new_pickled_task): @@ -55,7 +53,7 @@ def run(self): # a thread while loop cycle is atomic # thread safe locals: L = threading.local(), then L.foo="baz" django.setup() - self.logger.info('Worker Starts') + log.info('Worker Starts') done = False while not done: try: @@ -64,10 +62,13 @@ def run(self): done = True break + log.info('running task...') self.run_task(task) helpers.save_task_success(task) + log.info('...successfully') except Exception as e: - helpers.save_task_failed(task,e) + log.exception("...task failed") + helpers.save_task_failed(task, e) self.worker_queue = None - self.logger.warn('Worker stopped, %s tasks handled'%self.tasks_counter) + log.info('Worker stopped, {} tasks handled.'.format(self.tasks_counter)) diff --git a/django_leek/worker_manager.py b/django_leek/worker_manager.py index 11037c8..392dafa 100644 --- a/django_leek/worker_manager.py +++ b/django_leek/worker_manager.py @@ -3,7 +3,7 @@ def start(): global worker_thread - worker_thread = Worker(logger_name='main') + worker_thread = Worker() def put_task(task): From 68ba8180f18b20c33ed5454a83a91861d2d042f6 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 4 Sep 2018 14:56:26 +0200 Subject: [PATCH 058/109] ex: gitignoring dist/ files properly --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index be7ca2c..91059df 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # setuptools /build/ /dist/*.whl +/dist/*.gz *.egg-info/ # test app From b6fa5b397ff423baccd9320165d4625fb56770ee Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 4 Sep 2018 15:36:25 +0200 Subject: [PATCH 059/109] rm unused assignment --- django_leek/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index 8bda2ef..cf77bae 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -17,13 +17,13 @@ def serielize(task): def save_task_to_db(new_task): - t = models.QueuedTasks() pickled_task = serielize(new_task) t = models.QueuedTasks(pickled_task=pickled_task) t.save() new_task.db_id = t.id return new_task + def save_task_failed(task,exception): t = models.FailedTasks(task_id=task.db_id, exception=str(exception)) t.save() From 485a61dbffa4cdab435f9e06b3316c3e0e9cc52b Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 4 Sep 2018 15:48:12 +0200 Subject: [PATCH 060/109] fix leek "hanging" Previously, if a failed task could not be saved to the db, the worker thread bailed out causing there to be no active thread that processes new jobs. In Django tests, it's not easy to simlulate db failures, etc, so no test for this bugfix --- django_leek/worker.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index 4d83f9f..fa47c0b 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -68,7 +68,10 @@ def run(self): log.info('...successfully') except Exception as e: log.exception("...task failed") - helpers.save_task_failed(task, e) - + try: + helpers.save_task_failed(task, e) + except Exception: + log.exception("...could not update task as failed") + self.worker_queue = None log.info('Worker stopped, {} tasks handled.'.format(self.tasks_counter)) From fc042ca48d07d3d610d5dc27a8c59dae8e0ffa08 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 10:03:43 +0200 Subject: [PATCH 061/109] adding field for pool --- .../migrations/0002_queuedtasks_pool.py | 20 +++++++++++++++++++ django_leek/models.py | 1 + 2 files changed, 21 insertions(+) create mode 100644 django_leek/migrations/0002_queuedtasks_pool.py diff --git a/django_leek/migrations/0002_queuedtasks_pool.py b/django_leek/migrations/0002_queuedtasks_pool.py new file mode 100644 index 0000000..e656ad4 --- /dev/null +++ b/django_leek/migrations/0002_queuedtasks_pool.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-09-05 03:03 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='queuedtasks', + name='pool', + field=models.CharField(max_length=256, null=True), + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index e6ed0e8..46419e2 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -3,6 +3,7 @@ class QueuedTasks(models.Model): pickled_task = models.CharField(max_length=5000) #max row 65535 + pool = models.CharField(max_length=256, null=True) queued_on = models.DateTimeField(auto_now_add=True) From 99b1fcdce146152af9f68e027ada4caec14fa079 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 13:40:42 +0200 Subject: [PATCH 062/109] ex: reuse addr --- django_leek/management/commands/leek.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_leek/management/commands/leek.py b/django_leek/management/commands/leek.py index d46fd74..4c65e9a 100644 --- a/django_leek/management/commands/leek.py +++ b/django_leek/management/commands/leek.py @@ -25,6 +25,7 @@ def handle(self, *args, **options): host, port = _endpoint(cfg.get('bind', 'localhost:8002')) print('Listening on {port}'.format(port=port)) + socketserver.TCPServer.allow_reuse_address = True server = socketserver.TCPServer((host, port), TaskSocketServer) server.serve_forever() except KeyboardInterrupt: From 3d8336ebd9ff65680fbc69b6f5073998921e70bd Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 13:56:03 +0200 Subject: [PATCH 063/109] refactor: just transfer task id --- django_leek/api.py | 4 ++-- django_leek/helpers.py | 21 +++++++-------------- django_leek/server.py | 5 ++++- django_leek/worker.py | 6 +----- 4 files changed, 14 insertions(+), 22 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index dd03e03..3adf6a8 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -18,9 +18,9 @@ def push_task_to_queue(a_callable, *args, **kwargs): """Original API""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_task = Task(a_callable, *args, **kwargs) - new_task = helpers.save_task_to_db(new_task) # returns with db_id + queued_task = helpers.save_task_to_db(new_task) sock.connect((HOST, PORT)) - sock.send(helpers.serielize(new_task)) + sock.send("{}".format(queued_task.id).encode()) received = sock.recv(1024) sock.close() diff --git a/django_leek/helpers.py b/django_leek/helpers.py index cf77bae..64a4f18 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -7,8 +7,8 @@ def unpack(pickled_task): - new_task = pickle.loads(base64.b64decode(pickled_task)) - assert isinstance(new_task,Task) + new_task = pickle.loads(base64.b64decode(pickled_task)) + assert isinstance(new_task, Task) return new_task @@ -16,19 +16,12 @@ def serielize(task): return base64.b64encode(pickle.dumps(task)) +def load_task(task_id): + return models.QueuedTasks.objects.get(pk=task_id) + + def save_task_to_db(new_task): pickled_task = serielize(new_task) t = models.QueuedTasks(pickled_task=pickled_task) t.save() - new_task.db_id = t.id - return new_task - - -def save_task_failed(task,exception): - t = models.FailedTasks(task_id=task.db_id, exception=str(exception)) - t.save() - - -def save_task_success(task): - t = models.SuccessTasks(task_id=task.db_id) - t.save() + return t diff --git a/django_leek/server.py b/django_leek/server.py index 6128970..90b30e9 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -3,6 +3,7 @@ import threading from . import worker_manager +from .helpers import load_task log = logging.getLogger(__name__) @@ -33,7 +34,9 @@ def handle(self): # assume a serialized task log.info('Got a task') try: - response = worker_manager.put_task(data) + task_id = int(data.decode()) + queued_task = load_task(task_id=task_id) + response = worker_manager.put_task(queued_task.pickled_task) self.request.send(str(response).encode()) except Exception as e: log.exception("failed to queue task") diff --git a/django_leek/worker.py b/django_leek/worker.py index fa47c0b..883d566 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -64,14 +64,10 @@ def run(self): log.info('running task...') self.run_task(task) - helpers.save_task_success(task) + #helpers.save_task_success(task) log.info('...successfully') except Exception as e: log.exception("...task failed") - try: - helpers.save_task_failed(task, e) - except Exception: - log.exception("...could not update task as failed") self.worker_queue = None log.info('Worker stopped, {} tasks handled.'.format(self.tasks_counter)) From f2ba626118d53ff34ecaabd7760af9ba83f8f553 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 14:15:03 +0200 Subject: [PATCH 064/109] multipool support! --- django_leek/management/commands/leek.py | 2 - django_leek/server.py | 39 +++++++++++--- django_leek/worker.py | 68 ++++++------------------- django_leek/worker_manager.py | 22 -------- 4 files changed, 48 insertions(+), 83 deletions(-) delete mode 100644 django_leek/worker_manager.py diff --git a/django_leek/management/commands/leek.py b/django_leek/management/commands/leek.py index 4c65e9a..b825901 100644 --- a/django_leek/management/commands/leek.py +++ b/django_leek/management/commands/leek.py @@ -3,7 +3,6 @@ from django.core.management.base import BaseCommand, CommandError from django.conf import settings -from django_leek import worker_manager from django_leek.server import TaskSocketServer @@ -20,7 +19,6 @@ class Command(BaseCommand): def handle(self, *args, **options): try: - worker_manager.start() cfg = getattr(settings, 'LEEK', {}) host, port = _endpoint(cfg.get('bind', 'localhost:8002')) diff --git a/django_leek/server.py b/django_leek/server.py index 90b30e9..bb0a4ea 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,22 +1,32 @@ import logging import socketserver -import threading +import multiprocessing -from . import worker_manager from .helpers import load_task - +from . import worker +from . import helpers log = logging.getLogger(__name__) Dcommands = { - 'waiting': worker_manager.waiting, - 'handled': worker_manager.hanled, - 'stop': worker_manager.stop +# 'waiting': worker_manager.waiting, +# 'handled': worker_manager.hanled, +# 'stop': worker_manager.stop } +class Pool(object): + def __init__(self): + self.queue = multiprocessing.Queue() + self.worker = multiprocessing.Process(target=worker.target, args=(self.queue,)) + + class TaskSocketServer(socketserver.BaseRequestHandler): + DEFAULT_POOL = 'default' + # pools holds a mapping from pool names to process objects + pools = {DEFAULT_POOL: None} + def handle(self): try: data = self.request.recv(5000).strip() @@ -36,7 +46,22 @@ def handle(self): try: task_id = int(data.decode()) queued_task = load_task(task_id=task_id) - response = worker_manager.put_task(queued_task.pickled_task) + + # Ensure pool got a worker processing it + pool_name = queued_task.pool or self.DEFAULT_POOL + pool = self.pools[pool_name] + if pool is None or not pool.worker.is_alive(): + # Spawn new pool + log.info('Spawning new pool: {}'.format(pool_name)) + if pool is not None: + print(pool.worker.is_alive()) + self.pools[pool_name] = Pool() + self.pools[pool_name].worker.start() + + task = helpers.unpack(queued_task.pickled_task) + self.pools[pool_name].queue.put(task) + + response = (True, "sent") self.request.send(str(response).encode()) except Exception as e: log.exception("failed to queue task") diff --git a/django_leek/worker.py b/django_leek/worker.py index 883d566..446547c 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -16,58 +16,22 @@ log = logging.getLogger(__name__) -class Worker(threading.Thread): - def __init__(self): - threading.Thread.__init__(self, name="django-leek") - self.setDaemon(1) - self.worker_queue = queue.Queue() - self.tasks_counter = 0 - self.start() - - def put_task_on_queue(self, new_pickled_task): +def target(queue): + django.setup() + log.info('Worker Starts') + done = False + while not done: try: - new_task = helpers.unpack(new_pickled_task) - self.tasks_counter += 1 - self.worker_queue.put(new_task) - return True, "sent" - except Exception as e: - return False, "Worker: %s"%str(e) - - def run_task(self, task): - task.run() - - def stop_thread(self, timeout=None): - """Stop the thread and wait for it to end.""" - self.worker_queue.put(None) - self.join() - - def status_waiting(self): - return self.worker_queue.qsize() - - def status_handled(self): - # all, success & failes - return self.tasks_counter - - def run(self): - # the code until the while statement does NOT run atomicaly - # a thread while loop cycle is atomic - # thread safe locals: L = threading.local(), then L.foo="baz" - django.setup() - log.info('Worker Starts') - done = False - while not done: - try: - task = self.worker_queue.get() - if task is None: - done = True - break + task = queue.get() + if task is None: + done = True + break - log.info('running task...') - self.run_task(task) - #helpers.save_task_success(task) - log.info('...successfully') - except Exception as e: - log.exception("...task failed") + log.info('running task...') + task.run() + #helpers.save_task_success(task) + log.info('...successfully') + except Exception as e: + log.exception("...task failed") - self.worker_queue = None - log.info('Worker stopped, {} tasks handled.'.format(self.tasks_counter)) + log.info('Worker stopped') diff --git a/django_leek/worker_manager.py b/django_leek/worker_manager.py deleted file mode 100644 index 392dafa..0000000 --- a/django_leek/worker_manager.py +++ /dev/null @@ -1,22 +0,0 @@ -from .worker import Worker - - -def start(): - global worker_thread - worker_thread = Worker() - - -def put_task(task): - return worker_thread.put_task_on_queue(task) - - -def stop(): - return worker_thread.stop_thread() - - -def waiting(): - return worker_thread.status_waiting() - - -def hanled(): - return worker_thread.status_handled() From 4e5afbf7da10f456a6efa20475128cf7b99f4e8f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 14:52:11 +0200 Subject: [PATCH 065/109] refactor: rename misspelled symbol --- django_leek/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index 64a4f18..c93559e 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -12,7 +12,7 @@ def unpack(pickled_task): return new_task -def serielize(task): +def serialize(task): return base64.b64encode(pickle.dumps(task)) @@ -21,7 +21,7 @@ def load_task(task_id): def save_task_to_db(new_task): - pickled_task = serielize(new_task) + pickled_task = serialize(new_task) t = models.QueuedTasks(pickled_task=pickled_task) t.save() return t From 61f620d18dd0081ecb5b0180dd8be03591f56435 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 14:53:59 +0200 Subject: [PATCH 066/109] Development section --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 26e8cbb..7e20575 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,12 @@ With `django-leek` you can get up and running quickly The more complex distribut send_mail.offload(to='sync@leek.com') # now runs synchronously, like a normal function ``` +## Development +There is a test application you can play around with when developing on `django-leek`. Example: + +1. `./manage.sh test_app runserver` - Starts the test app +2. `./manage.sh test_app leek` - Starts a leek instance for the test app +3. `./manage.sh django_leek test` - Run test suite. ## Technical overview In a nutshell, a python SocketServer runs in the background, listening on a tcp socket. SocketServer gets the request to run a task from it's socket, puts the task on a Queue. A Worker thread picks tasks from this Queue, and runs the tasks one by one. From c3e9bebc02cd142f4a99ed8374ab45957d308c47 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 14:58:10 +0200 Subject: [PATCH 067/109] refactor: Using built-in `partial` over duplicated `Task` --- django_leek/api.py | 5 ++--- django_leek/helpers.py | 4 ++-- django_leek/task.py | 10 ---------- django_leek/tests.py | 4 ++-- django_leek/worker.py | 2 +- 5 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 django_leek/task.py diff --git a/django_leek/api.py b/django_leek/api.py index 3adf6a8..2b514ee 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -1,6 +1,5 @@ import socket -from functools import wraps -from .task import Task +from functools import wraps, partial from . import helpers from .settings import HOST, PORT @@ -17,7 +16,7 @@ def _offload(*args, **kwargs): def push_task_to_queue(a_callable, *args, **kwargs): """Original API""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - new_task = Task(a_callable, *args, **kwargs) + new_task = partial(a_callable, *args, **kwargs) queued_task = helpers.save_task_to_db(new_task) sock.connect((HOST, PORT)) sock.send("{}".format(queued_task.id).encode()) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index c93559e..55d9d99 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -1,14 +1,14 @@ import datetime import pickle import base64 +from functools import partial from . import models -from .task import Task def unpack(pickled_task): new_task = pickle.loads(base64.b64decode(pickled_task)) - assert isinstance(new_task, Task) + assert isinstance(new_task, partial) return new_task diff --git a/django_leek/task.py b/django_leek/task.py deleted file mode 100644 index 7122389..0000000 --- a/django_leek/task.py +++ /dev/null @@ -1,10 +0,0 @@ -class Task(object): - def __init__(self, a_callable, *args, **kwargs): - assert callable(a_callable) - self.task_callable = a_callable - self.args = args - self.kwargs = kwargs - self.db_id = None - - def run(self): - self.task_callable(*self.args, **self.kwargs) diff --git a/django_leek/tests.py b/django_leek/tests.py index 289dbe7..b5aec1d 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -2,12 +2,12 @@ import pickle from unittest.mock import patch, MagicMock import socketserver +from functools import partial from django.test import TestCase from django.core.management import call_command from django_leek.server import TaskSocketServer -from django_leek.task import Task from django_leek import helpers @@ -45,7 +45,7 @@ def test_recv_error(self): self.act() def test_task(self): - task = helpers.save_task_to_db(Task(f)) + task = helpers.save_task_to_db(partial(f)) self._request(base64.b64encode(pickle.dumps(task))) self.act() self.assertEqual(self._response(), b"(True, 'sent')") diff --git a/django_leek/worker.py b/django_leek/worker.py index 446547c..304b555 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -28,7 +28,7 @@ def target(queue): break log.info('running task...') - task.run() + task() #helpers.save_task_success(task) log.info('...successfully') except Exception as e: From 49396bed96536f5327a90f7814b852a8b5d1851f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 15:11:59 +0200 Subject: [PATCH 068/109] rm: happy path test --- django_leek/tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/django_leek/tests.py b/django_leek/tests.py index b5aec1d..960bcee 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -43,9 +43,3 @@ def act(self): def test_recv_error(self): self._request(OSError('Nuclear Winter')) self.act() - - def test_task(self): - task = helpers.save_task_to_db(partial(f)) - self._request(base64.b64encode(pickle.dumps(task))) - self.act() - self.assertEqual(self._response(), b"(True, 'sent')") From 4e8da7eda8a8e5b91edbc1496f29d85a413e25b8 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 15:17:47 +0200 Subject: [PATCH 069/109] Defaults to run on pool name as wraped function --- django_leek/api.py | 9 +++++---- django_leek/helpers.py | 4 ++-- django_leek/server.py | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index 2b514ee..352809d 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -5,19 +5,20 @@ class Leek(object): - def task(self, f): + def task(self, f, pool=None): + pool_name = pool or f.__name__ @wraps(f) def _offload(*args, **kwargs): - return push_task_to_queue(f, *args, **kwargs) + return push_task_to_queue(f, pool_name=pool_name, *args, **kwargs) f.offload = _offload return f -def push_task_to_queue(a_callable, *args, **kwargs): +def push_task_to_queue(a_callable, pool_name=None, *args, **kwargs): """Original API""" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_task = partial(a_callable, *args, **kwargs) - queued_task = helpers.save_task_to_db(new_task) + queued_task = helpers.save_task_to_db(new_task, pool_name) sock.connect((HOST, PORT)) sock.send("{}".format(queued_task.id).encode()) received = sock.recv(1024) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index 55d9d99..b8633a1 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -20,8 +20,8 @@ def load_task(task_id): return models.QueuedTasks.objects.get(pk=task_id) -def save_task_to_db(new_task): +def save_task_to_db(new_task, pool_name): pickled_task = serialize(new_task) - t = models.QueuedTasks(pickled_task=pickled_task) + t = models.QueuedTasks(pickled_task=pickled_task, pool=pool_name) t.save() return t diff --git a/django_leek/server.py b/django_leek/server.py index bb0a4ea..7b0bc88 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -25,7 +25,7 @@ def __init__(self): class TaskSocketServer(socketserver.BaseRequestHandler): DEFAULT_POOL = 'default' # pools holds a mapping from pool names to process objects - pools = {DEFAULT_POOL: None} + pools = {} def handle(self): try: @@ -49,10 +49,10 @@ def handle(self): # Ensure pool got a worker processing it pool_name = queued_task.pool or self.DEFAULT_POOL - pool = self.pools[pool_name] + pool = self.pools.get(pool_name) if pool is None or not pool.worker.is_alive(): # Spawn new pool - log.info('Spawning new pool: {}'.format(pool_name)) + log.info('Spawning new pool: {}'.format(pool_name)) if pool is not None: print(pool.worker.is_alive()) self.pools[pool_name] = Pool() From 702a27e85469935ee33d782ba9bfa2204b54ca61 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 15:37:30 +0200 Subject: [PATCH 070/109] refactor: rm dead code --- django_leek/server.py | 6 +----- django_leek/tests.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 7b0bc88..0a6ca1b 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -9,11 +9,7 @@ log = logging.getLogger(__name__) -Dcommands = { -# 'waiting': worker_manager.waiting, -# 'handled': worker_manager.hanled, -# 'stop': worker_manager.stop -} +Dcommands = {} class Pool(object): diff --git a/django_leek/tests.py b/django_leek/tests.py index 960bcee..1ed5dc0 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -2,7 +2,6 @@ import pickle from unittest.mock import patch, MagicMock import socketserver -from functools import partial from django.test import TestCase from django.core.management import call_command From c2888b1dab8882ac004620bf42ac4e9d4b1ec252 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 5 Sep 2018 15:49:16 +0200 Subject: [PATCH 071/109] 0.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4a1fcd0..76632ee 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.6', + version='0.7', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 6a54f276d0e3cc52c9cdad13b7b32ce2cae8b8f3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 09:26:11 +0200 Subject: [PATCH 072/109] Revert "rm: happy path test" This reverts commit 49396bed96536f5327a90f7814b852a8b5d1851f. --- django_leek/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django_leek/tests.py b/django_leek/tests.py index 1ed5dc0..63faa2c 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -42,3 +42,9 @@ def act(self): def test_recv_error(self): self._request(OSError('Nuclear Winter')) self.act() + + def test_task(self): + task = helpers.save_task_to_db(partial(f)) + self._request(base64.b64encode(pickle.dumps(task))) + self.act() + self.assertEqual(self._response(), b"(True, 'sent')") From c1c2c982e8e6b0e1992e29af94ab2eb1ba739732 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 10:37:18 +0200 Subject: [PATCH 073/109] build: fix happy-path test --- django_leek/server.py | 4 ++++ django_leek/tests.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 0a6ca1b..55a3fda 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -67,3 +67,7 @@ def handle(self): except OSError as e: # in case of network error, just log log.exception("network error") + + def finish(self): + for pool in self.pools.values(): + pool.queue.put(None) diff --git a/django_leek/tests.py b/django_leek/tests.py index 63faa2c..7cf52cc 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,4 +1,5 @@ import base64 +from functools import partial import pickle from unittest.mock import patch, MagicMock import socketserver @@ -44,7 +45,7 @@ def test_recv_error(self): self.act() def test_task(self): - task = helpers.save_task_to_db(partial(f)) - self._request(base64.b64encode(pickle.dumps(task))) + task = helpers.save_task_to_db(partial(f), 'pool_name') + self._request(str(task.id).encode()) self.act() self.assertEqual(self._response(), b"(True, 'sent')") From 195436583f702742438459d7eb8361265fc43b8f Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 10:46:07 +0200 Subject: [PATCH 074/109] rm random printout --- django_leek/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 55a3fda..e6e6c62 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -49,8 +49,6 @@ def handle(self): if pool is None or not pool.worker.is_alive(): # Spawn new pool log.info('Spawning new pool: {}'.format(pool_name)) - if pool is not None: - print(pool.worker.is_alive()) self.pools[pool_name] = Pool() self.pools[pool_name].worker.start() From c14e64f7715c31db08551796586496bd5ebf55d1 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 10:46:25 +0200 Subject: [PATCH 075/109] fix: passing pool_name by position --- django_leek/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_leek/api.py b/django_leek/api.py index 352809d..a3c9d4c 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -9,7 +9,7 @@ def task(self, f, pool=None): pool_name = pool or f.__name__ @wraps(f) def _offload(*args, **kwargs): - return push_task_to_queue(f, pool_name=pool_name, *args, **kwargs) + return push_task_to_queue(f, pool_name, *args, **kwargs) f.offload = _offload return f From d161779575d70035ad1a048306c1bd8052b171c3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 11:01:06 +0200 Subject: [PATCH 076/109] fix: problem with args/kwargs --- django_leek/api.py | 5 +++-- test_app/app.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index a3c9d4c..9b71570 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -9,13 +9,14 @@ def task(self, f, pool=None): pool_name = pool or f.__name__ @wraps(f) def _offload(*args, **kwargs): - return push_task_to_queue(f, pool_name, *args, **kwargs) + return push_task_to_queue(f, pool_name=pool_name, *args, **kwargs) f.offload = _offload return f -def push_task_to_queue(a_callable, pool_name=None, *args, **kwargs): +def push_task_to_queue(a_callable, *args, **kwargs): """Original API""" + pool_name = kwargs.pop('pool_name', None) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) new_task = partial(a_callable, *args, **kwargs) queued_task = helpers.save_task_to_db(new_task, pool_name) diff --git a/test_app/app.py b/test_app/app.py index 3a66e98..d25ff4b 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -26,7 +26,8 @@ def index(request): hello(to='sync') # Run async - hello.offload(to='new') + hello.offload(to='kwargs') + hello.offload('args') push_task_to_queue(hello, to='old') return render(request, 'index.html', {'message': '✓ task queued'}) From d451c49d4e0f1bcbb5b2bf7af38b5cb05ab72f00 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Fri, 7 Sep 2018 11:10:35 +0200 Subject: [PATCH 077/109] v8.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 76632ee..7e9d112 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.7', + version='0.8', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 302bba600efe02d6d638790041e1906fd421e4e6 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 10 Sep 2018 11:55:21 +0200 Subject: [PATCH 078/109] reinstating `Task` ...after suspected pickling problems with `partial` --- django_leek/api.py | 15 +++++++++++++-- django_leek/helpers.py | 3 +-- django_leek/tests.py | 5 ++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index 9b71570..f615843 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -1,5 +1,5 @@ import socket -from functools import wraps, partial +from functools import wraps from . import helpers from .settings import HOST, PORT @@ -14,11 +14,22 @@ def _offload(*args, **kwargs): return f +class Task(object): + def __init__(self, a_callable, *args, **kwargs): + assert callable(a_callable) + self.task_callable = a_callable + self.args = args + self.kwargs = kwargs + + def __call__(self): + self.task_callable(*self.args, **self.kwargs) + + def push_task_to_queue(a_callable, *args, **kwargs): """Original API""" pool_name = kwargs.pop('pool_name', None) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - new_task = partial(a_callable, *args, **kwargs) + new_task = Task(a_callable, *args, **kwargs) queued_task = helpers.save_task_to_db(new_task, pool_name) sock.connect((HOST, PORT)) sock.send("{}".format(queued_task.id).encode()) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index b8633a1..70d8253 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -1,14 +1,13 @@ import datetime import pickle import base64 -from functools import partial from . import models def unpack(pickled_task): new_task = pickle.loads(base64.b64decode(pickled_task)) - assert isinstance(new_task, partial) + #assert isinstance(new_task, partial) return new_task diff --git a/django_leek/tests.py b/django_leek/tests.py index 7cf52cc..9c6eb26 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,5 +1,4 @@ import base64 -from functools import partial import pickle from unittest.mock import patch, MagicMock import socketserver @@ -8,7 +7,7 @@ from django.core.management import call_command from django_leek.server import TaskSocketServer -from django_leek import helpers +from django_leek import helpers, api @patch.object(socketserver.TCPServer, 'serve_forever') @@ -45,7 +44,7 @@ def test_recv_error(self): self.act() def test_task(self): - task = helpers.save_task_to_db(partial(f), 'pool_name') + task = helpers.save_task_to_db(api.Task(f), 'pool_name') self._request(str(task.id).encode()) self.act() self.assertEqual(self._response(), b"(True, 'sent')") From 9e9e67d260f122ed9f87d3bbaecc36e9470a2286 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 10 Sep 2018 17:32:16 +0200 Subject: [PATCH 079/109] store pickles in binaryfield --- .../migrations/0003_auto_20180910_1028.py | 20 +++++++++++++++++++ django_leek/models.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 django_leek/migrations/0003_auto_20180910_1028.py diff --git a/django_leek/migrations/0003_auto_20180910_1028.py b/django_leek/migrations/0003_auto_20180910_1028.py new file mode 100644 index 0000000..5d09139 --- /dev/null +++ b/django_leek/migrations/0003_auto_20180910_1028.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2018-09-10 10:28 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0002_queuedtasks_pool'), + ] + + operations = [ + migrations.AlterField( + model_name='queuedtasks', + name='pickled_task', + field=models.BinaryField(max_length=5000), + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index 46419e2..b3f9ff0 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -2,7 +2,7 @@ class QueuedTasks(models.Model): - pickled_task = models.CharField(max_length=5000) #max row 65535 + pickled_task = models.BinaryField(max_length=5000) #max row 65535 pool = models.CharField(max_length=256, null=True) queued_on = models.DateTimeField(auto_now_add=True) From f79998bb8c7a5b4a6d40309d23999222e15a7fc4 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Mon, 10 Sep 2018 17:46:28 +0200 Subject: [PATCH 080/109] v0.9 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e9d112..30b37c5 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.8', + version='0.9', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 9574d44f102c9b8704a8e01c5e416bcce0fffb48 Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 26 Sep 2018 14:20:34 +0200 Subject: [PATCH 081/109] Fixed problem with database connection --- django_leek/worker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_leek/worker.py b/django_leek/worker.py index 304b555..26b86d4 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -29,7 +29,9 @@ def target(queue): log.info('running task...') task() - #helpers.save_task_success(task) + # workaround to solve problems with django + psycopg2 + # solution found here: https://stackoverflow.com/a/36580629/10385696 + django.db.connection.close() log.info('...successfully') except Exception as e: log.exception("...task failed") From e06e712c7103879572ab79296fdbed224b8c43f3 Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 26 Sep 2018 14:23:48 +0200 Subject: [PATCH 082/109] Removed unused imports and PEP8 whitespaces --- django_leek/api.py | 1 + django_leek/helpers.py | 2 -- django_leek/run_failed_tasks.py | 2 -- django_leek/tests.py | 4 ++-- django_leek/worker.py | 10 ---------- 5 files changed, 3 insertions(+), 16 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index f615843..8d185ff 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -7,6 +7,7 @@ class Leek(object): def task(self, f, pool=None): pool_name = pool or f.__name__ + @wraps(f) def _offload(*args, **kwargs): return push_task_to_queue(f, pool_name=pool_name, *args, **kwargs) diff --git a/django_leek/helpers.py b/django_leek/helpers.py index 70d8253..f7cd175 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -1,4 +1,3 @@ -import datetime import pickle import base64 @@ -7,7 +6,6 @@ def unpack(pickled_task): new_task = pickle.loads(base64.b64decode(pickled_task)) - #assert isinstance(new_task, partial) return new_task diff --git a/django_leek/run_failed_tasks.py b/django_leek/run_failed_tasks.py index b7a3ef6..ae52612 100644 --- a/django_leek/run_failed_tasks.py +++ b/django_leek/run_failed_tasks.py @@ -1,8 +1,6 @@ import pickle import base64 import os -import sys - # environ settings variable, should be the same as in manage.py os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") os.environ["DJANGO_SETTINGS_MODULE"] = "mysite.settings" diff --git a/django_leek/tests.py b/django_leek/tests.py index 9c6eb26..74526b0 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,5 +1,3 @@ -import base64 -import pickle from unittest.mock import patch, MagicMock import socketserver @@ -20,9 +18,11 @@ def test_keyboard_interrupt(self, serve_forever): serve_forever.side_effect = KeyboardInterrupt call_command('leek') + def f(): pass + class TestServer(TestCase): def setUp(self): self.request = MagicMock() diff --git a/django_leek/worker.py b/django_leek/worker.py index 26b86d4..cf651bf 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -1,18 +1,8 @@ # Based on the Worker class from Python in a nutshell, by Alex Martelli import logging -import os -import sys -import threading -import queue - import django - -from . import settings -from . import helpers - - log = logging.getLogger(__name__) From 51a591aa8590616421fc1a5c84c510d411223116 Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 26 Sep 2018 14:38:47 +0200 Subject: [PATCH 083/109] Changed package version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 30b37c5..4dcf456 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.9', + version='0.9.1', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 7150090e93a54fd7dbcde3c4b585925bc59c873b Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Tue, 2 Oct 2018 15:01:42 +0200 Subject: [PATCH 084/109] close all connection before loading tasks --- django_leek/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_leek/server.py b/django_leek/server.py index e6e6c62..fa4c4d0 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -5,6 +5,7 @@ from .helpers import load_task from . import worker from . import helpers +import django log = logging.getLogger(__name__) @@ -41,6 +42,7 @@ def handle(self): log.info('Got a task') try: task_id = int(data.decode()) + django.db.connections.close_all() queued_task = load_task(task_id=task_id) # Ensure pool got a worker processing it From 939009e77fd58d910113fcdbe1ad595deca22cbc Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 3 Oct 2018 10:35:27 +0200 Subject: [PATCH 085/109] README updated with instructions on how to deploy a new version --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e20575..c1b1501 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,20 @@ The SQL to delete these tasks: In a similar way, delete the failed tasks. You can run a cron script, or other script, to purge the tasks. +## Release a new version +1. Checkout master branch +2. Make sure virtual environment is activated. `source venv/bin/activate` +3, Make sure version in `setup.py` is correct. `grep version setup.py` +4. Make sure setuptools, twine, and wheel are installed and up to date + + pip install "setuptools>=38.6.0" "twine>=1.11.0" "wheel>=0.31.0" + +5. Clean out any old dist packages. `rm -r dist/` +6. Build source and wheel dists. `python setup.py sdist bdist_wheel` +7. Upload to PyPI `twine upload dist/*` +8. Profit! ## Authors -Aviah and Samuel Carlsson +Aviah, Silvia Scalisi and Samuel Carlsson See [contributors]( https://github.com/Volumental/django-leek/graphs/contributors) for full list. From 73029abab9400cb05c98a7e960cdbdc07e002db3 Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 3 Oct 2018 10:36:03 +0200 Subject: [PATCH 086/109] New version in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4dcf456..7aa80a2 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.9.1', + version='0.9.2', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 80f86210f329770c20b9d70d0577b5eee30fe6f1 Mon Sep 17 00:00:00 2001 From: Silvia Scalisi Date: Wed, 3 Oct 2018 10:46:05 +0200 Subject: [PATCH 087/109] Fixed typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1b1501..e4b1bda 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ You can run a cron script, or other script, to purge the tasks. ## Release a new version 1. Checkout master branch 2. Make sure virtual environment is activated. `source venv/bin/activate` -3, Make sure version in `setup.py` is correct. `grep version setup.py` +3. Make sure version in `setup.py` is correct. `grep version setup.py` 4. Make sure setuptools, twine, and wheel are installed and up to date pip install "setuptools>=38.6.0" "twine>=1.11.0" "wheel>=0.31.0" From 28072859b3929094d492599442e99ccee6342d91 Mon Sep 17 00:00:00 2001 From: Niklas Karlsson Date: Wed, 14 Nov 2018 12:43:50 +0100 Subject: [PATCH 088/109] Force new db connection to be created for forkes --- django_leek/server.py | 2 ++ django_leek/worker.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/django_leek/server.py b/django_leek/server.py index fa4c4d0..47f0ca2 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -42,6 +42,8 @@ def handle(self): log.info('Got a task') try: task_id = int(data.decode()) + + # Connection are closed by tasks, force it to reconnect django.db.connections.close_all() queued_task = load_task(task_id=task_id) diff --git a/django_leek/worker.py b/django_leek/worker.py index cf651bf..10bf769 100644 --- a/django_leek/worker.py +++ b/django_leek/worker.py @@ -18,6 +18,10 @@ def target(queue): break log.info('running task...') + + # Force this forked process to create its own db connection + django.db.connection.close() + task() # workaround to solve problems with django + psycopg2 # solution found here: https://stackoverflow.com/a/36580629/10385696 From f725b39b3610e51242b4ed5d8dbc1babe8ee63af Mon Sep 17 00:00:00 2001 From: Niklas Karlsson Date: Wed, 14 Nov 2018 13:11:57 +0100 Subject: [PATCH 089/109] Ticked up version number --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7aa80a2..3c606cb 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.9.2', + version='0.9.3', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, @@ -34,4 +34,4 @@ 'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', ], -) \ No newline at end of file +) From 5b838a38e4b7cd915810a6add22eb8bea45c4dad Mon Sep 17 00:00:00 2001 From: Daniel Zavala Svensson Date: Mon, 19 Nov 2018 17:25:47 +0100 Subject: [PATCH 090/109] Fixed erroneous syntax in README.md In the django settings INSTALLED_APPS, it should be 'django_leek' not 'django-leek' and the command to run the leek server is 'python manage.py leek' not 'python manage.py runleek'. This fixes #12 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4b1bda..5ac0a42 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ With `django-leek` you can get up and running quickly The more complex distribut $ pip install django-leek ```` -2. Add `django-leek` to `INSTALLED_APPS` in your `settings.py` file. +2. Add `django_leek` to `INSTALLED_APPS` in your `settings.py` file. 3. Create tables needed @@ -39,7 +39,7 @@ With `django-leek` you can get up and running quickly The more complex distribut 4. Make sure the django-leek server is running. ```bash - $ python manage.py runleek + $ python manage.py leek ``` 5. Go nuts From 314038e4ad6f1a3a9d8b887febc8ff01dba9cdc7 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 4 Mar 2020 18:19:42 +0100 Subject: [PATCH 091/109] refactor: mv function --- django_leek/server.py | 30 ++++++++++++++++++++++++++++-- django_leek/worker.py | 33 --------------------------------- 2 files changed, 28 insertions(+), 35 deletions(-) delete mode 100644 django_leek/worker.py diff --git a/django_leek/server.py b/django_leek/server.py index 47f0ca2..99f5cc3 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -3,7 +3,6 @@ import multiprocessing from .helpers import load_task -from . import worker from . import helpers import django @@ -13,10 +12,37 @@ Dcommands = {} +def target(queue): + django.setup() + log.info('Worker Starts') + done = False + while not done: + try: + task = queue.get() + if task is None: + done = True + break + + log.info('running task...') + + # Force this forked process to create its own db connection + django.db.connection.close() + + task() + # workaround to solve problems with django + psycopg2 + # solution found here: https://stackoverflow.com/a/36580629/10385696 + django.db.connection.close() + log.info('...successfully') + except Exception as e: + log.exception("...task failed") + + log.info('Worker stopped') + + class Pool(object): def __init__(self): self.queue = multiprocessing.Queue() - self.worker = multiprocessing.Process(target=worker.target, args=(self.queue,)) + self.worker = multiprocessing.Process(target=target, args=(self.queue,)) class TaskSocketServer(socketserver.BaseRequestHandler): diff --git a/django_leek/worker.py b/django_leek/worker.py deleted file mode 100644 index 10bf769..0000000 --- a/django_leek/worker.py +++ /dev/null @@ -1,33 +0,0 @@ -# Based on the Worker class from Python in a nutshell, by Alex Martelli -import logging - -import django - -log = logging.getLogger(__name__) - - -def target(queue): - django.setup() - log.info('Worker Starts') - done = False - while not done: - try: - task = queue.get() - if task is None: - done = True - break - - log.info('running task...') - - # Force this forked process to create its own db connection - django.db.connection.close() - - task() - # workaround to solve problems with django + psycopg2 - # solution found here: https://stackoverflow.com/a/36580629/10385696 - django.db.connection.close() - log.info('...successfully') - except Exception as e: - log.exception("...task failed") - - log.info('Worker stopped') From 6f668315208cbd17e1c4648e017a054f91d084b4 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 4 Mar 2020 18:36:17 +0100 Subject: [PATCH 092/109] combined task table --- django_leek/helpers.py | 4 +- .../migrations/0004_auto_20200304_1134.py | 37 +++++++++++++++++++ django_leek/models.py | 19 +++------- 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 django_leek/migrations/0004_auto_20200304_1134.py diff --git a/django_leek/helpers.py b/django_leek/helpers.py index f7cd175..d09220e 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -14,11 +14,11 @@ def serialize(task): def load_task(task_id): - return models.QueuedTasks.objects.get(pk=task_id) + return models.Task.objects.get(pk=task_id) def save_task_to_db(new_task, pool_name): pickled_task = serialize(new_task) - t = models.QueuedTasks(pickled_task=pickled_task, pool=pool_name) + t = models.Task(pickled_task=pickled_task, pool=pool_name) t.save() return t diff --git a/django_leek/migrations/0004_auto_20200304_1134.py b/django_leek/migrations/0004_auto_20200304_1134.py new file mode 100644 index 0000000..83bb7b6 --- /dev/null +++ b/django_leek/migrations/0004_auto_20200304_1134.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2020-03-04 11:34 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0003_auto_20180910_1028'), + ] + + operations = [ + migrations.CreateModel( + name='Task', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pickled_task', models.BinaryField(max_length=4096)), + ('pool', models.CharField(max_length=256, null=True)), + ('queued_on', models.DateTimeField(auto_now_add=True)), + ('started_at', models.DateTimeField(null=True)), + ('finished_at', models.DateTimeField(null=True)), + ('exception', models.CharField(max_length=2048, null=True)), + ('pickled_return', models.BinaryField(max_length=4096, null=True)), + ], + ), + migrations.DeleteModel( + name='FailedTasks', + ), + migrations.DeleteModel( + name='QueuedTasks', + ), + migrations.DeleteModel( + name='SuccessTasks', + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index b3f9ff0..eac5789 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -1,18 +1,11 @@ from django.db import models -class QueuedTasks(models.Model): - pickled_task = models.BinaryField(max_length=5000) #max row 65535 +class Task(models.Model): + pickled_task = models.BinaryField(max_length=4096) pool = models.CharField(max_length=256, null=True) queued_on = models.DateTimeField(auto_now_add=True) - - -class SuccessTasks(models.Model): - task_id = models.IntegerField() - saved_on = models.DateTimeField(auto_now_add=True) - - -class FailedTasks(models.Model): - task_id = models.IntegerField() - exception = models.CharField(max_length=2048) - saved_on = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + exception = models.CharField(max_length=2048, null=True) + pickled_return = models.BinaryField(max_length=4096, null=True) From 366b4d05d22a407582334e879fb80dc9d73eafd3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 4 Mar 2020 18:36:23 +0100 Subject: [PATCH 093/109] rm obsolete script --- django_leek/run_failed_tasks.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 django_leek/run_failed_tasks.py diff --git a/django_leek/run_failed_tasks.py b/django_leek/run_failed_tasks.py deleted file mode 100644 index ae52612..0000000 --- a/django_leek/run_failed_tasks.py +++ /dev/null @@ -1,19 +0,0 @@ -import pickle -import base64 -import os -# environ settings variable, should be the same as in manage.py -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") -os.environ["DJANGO_SETTINGS_MODULE"] = "mysite.settings" -import django -django.setup() - - -# edit to the correct mysite.tasks_queue path -from .models import FailedTasks, QueuedTasks - - -Lfailed_tasks_id = FailedTasks.objects.values_list("task_id", flat=True) -tasks = QueuedTasks.objects.filter(pk__in=Lfailed_tasks_id) -for r in tasks: - task = pickle.loads(base64.b64decode(r.pickled_task)) - task.run() From 1c70540c02d283bb52bdbc5ab51831d509f4f785 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 4 Mar 2020 18:37:04 +0100 Subject: [PATCH 094/109] refactor: rm dict --- django_leek/server.py | 69 ++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 99f5cc3..af2c9dc 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -9,9 +9,6 @@ log = logging.getLogger(__name__) -Dcommands = {} - - def target(queue): django.setup() log.info('Worker Starts') @@ -53,44 +50,34 @@ class TaskSocketServer(socketserver.BaseRequestHandler): def handle(self): try: data = self.request.recv(5000).strip() - if data in Dcommands.keys(): - log.info('Got command: "{}"'.format(data)) - try: - worker_response = Dcommands[data]() - response = (True, worker_response.encode(),) - self.request.send(str(response).encode()) - except Exception as e: - log.exception("command failed") - response = (False, "TaskServer Command: {}".format(e).encode(),) - self.request.send(str(response).encode()) - else: - # assume a serialized task - log.info('Got a task') - try: - task_id = int(data.decode()) - - # Connection are closed by tasks, force it to reconnect - django.db.connections.close_all() - queued_task = load_task(task_id=task_id) - - # Ensure pool got a worker processing it - pool_name = queued_task.pool or self.DEFAULT_POOL - pool = self.pools.get(pool_name) - if pool is None or not pool.worker.is_alive(): - # Spawn new pool - log.info('Spawning new pool: {}'.format(pool_name)) - self.pools[pool_name] = Pool() - self.pools[pool_name].worker.start() - - task = helpers.unpack(queued_task.pickled_task) - self.pools[pool_name].queue.put(task) - - response = (True, "sent") - self.request.send(str(response).encode()) - except Exception as e: - log.exception("failed to queue task") - response = (False, "TaskServer Put: {}".format(e).encode(),) - self.request.send(str(response).encode()) + + # assume a serialized task + log.info('Got a task') + try: + task_id = int(data.decode()) + + # Connection are closed by tasks, force it to reconnect + django.db.connections.close_all() + queued_task = load_task(task_id=task_id) + + # Ensure pool got a worker processing it + pool_name = queued_task.pool or self.DEFAULT_POOL + pool = self.pools.get(pool_name) + if pool is None or not pool.worker.is_alive(): + # Spawn new pool + log.info('Spawning new pool: {}'.format(pool_name)) + self.pools[pool_name] = Pool() + self.pools[pool_name].worker.start() + + task = helpers.unpack(queued_task.pickled_task) + self.pools[pool_name].queue.put(task) + + response = (True, "sent") + self.request.send(str(response).encode()) + except Exception as e: + log.exception("failed to queue task") + response = (False, "TaskServer Put: {}".format(e).encode(),) + self.request.send(str(response).encode()) except OSError as e: # in case of network error, just log From 6abd8363f4ca654e325ab716cd73d7530a86b4a3 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 4 Mar 2020 18:45:41 +0100 Subject: [PATCH 095/109] update status as tasks are ran --- django_leek/api.py | 2 +- django_leek/helpers.py | 2 +- django_leek/server.py | 46 ++++++++++++++++++++++++++---------------- test_app/app.py | 1 + 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/django_leek/api.py b/django_leek/api.py index 8d185ff..08be757 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -23,7 +23,7 @@ def __init__(self, a_callable, *args, **kwargs): self.kwargs = kwargs def __call__(self): - self.task_callable(*self.args, **self.kwargs) + return self.task_callable(*self.args, **self.kwargs) def push_task_to_queue(a_callable, *args, **kwargs): diff --git a/django_leek/helpers.py b/django_leek/helpers.py index d09220e..dc34224 100644 --- a/django_leek/helpers.py +++ b/django_leek/helpers.py @@ -13,7 +13,7 @@ def serialize(task): return base64.b64encode(pickle.dumps(task)) -def load_task(task_id): +def load_task(task_id) -> models.Task: return models.Task.objects.get(pk=task_id) diff --git a/django_leek/server.py b/django_leek/server.py index af2c9dc..86b427c 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,3 +1,4 @@ +from datetime import datetime import logging import socketserver import multiprocessing @@ -14,24 +15,36 @@ def target(queue): log.info('Worker Starts') done = False while not done: - try: - task = queue.get() - if task is None: - done = True - break - - log.info('running task...') + task_id = queue.get() + if task_id is None: + done = True + break + + log.info('running task...') - # Force this forked process to create its own db connection - django.db.connection.close() + # Force this forked process to create its own db connection + django.db.connection.close() + + task = load_task(task_id=task_id) + t = helpers.unpack(task.pickled_task) + try: + task.started_at = datetime.now() + task.save() + r = t() + task.finished_at = datetime.now() + task.pickled_return = helpers.serialize(r) + task.save() - task() - # workaround to solve problems with django + psycopg2 - # solution found here: https://stackoverflow.com/a/36580629/10385696 - django.db.connection.close() log.info('...successfully') except Exception as e: log.exception("...task failed") + task.finished_at = datetime.now() + task.exception = helpers.serialize(e) + task.save() + + # workaround to solve problems with django + psycopg2 + # solution found here: https://stackoverflow.com/a/36580629/10385696 + django.db.connection.close() log.info('Worker stopped') @@ -58,10 +71,10 @@ def handle(self): # Connection are closed by tasks, force it to reconnect django.db.connections.close_all() - queued_task = load_task(task_id=task_id) + task = load_task(task_id=task_id) # Ensure pool got a worker processing it - pool_name = queued_task.pool or self.DEFAULT_POOL + pool_name = task.pool or self.DEFAULT_POOL pool = self.pools.get(pool_name) if pool is None or not pool.worker.is_alive(): # Spawn new pool @@ -69,8 +82,7 @@ def handle(self): self.pools[pool_name] = Pool() self.pools[pool_name].worker.start() - task = helpers.unpack(queued_task.pickled_task) - self.pools[pool_name].queue.put(task) + self.pools[pool_name].queue.put(task_id) response = (True, "sent") self.request.send(str(response).encode()) diff --git a/test_app/app.py b/test_app/app.py index d25ff4b..7645a2e 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -18,6 +18,7 @@ def hello(to): person.save() print('Hello {}!'.format(to)) + return 'wow' def index(request): From e9e70e7d5517a7f369e1214eafa807cd9db0b78e Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Thu, 5 Mar 2020 15:03:54 +0100 Subject: [PATCH 096/109] function for querying task --- django_leek/api.py | 10 +++- .../migrations/0005_auto_20200305_0753.py | 20 +++++++ django_leek/models.py | 18 ++++++- django_leek/server.py | 15 ++++-- test_app/app.py | 52 +++++++++++++++---- test_app/templates/index.html | 39 ++++++++++++++ 6 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 django_leek/migrations/0005_auto_20200305_0753.py diff --git a/django_leek/api.py b/django_leek/api.py index 08be757..f8c51f8 100644 --- a/django_leek/api.py +++ b/django_leek/api.py @@ -1,5 +1,8 @@ import socket from functools import wraps +import json + +from . import models from . import helpers from .settings import HOST, PORT @@ -21,7 +24,7 @@ def __init__(self, a_callable, *args, **kwargs): self.task_callable = a_callable self.args = args self.kwargs = kwargs - + def __call__(self): return self.task_callable(*self.args, **self.kwargs) @@ -36,5 +39,8 @@ def push_task_to_queue(a_callable, *args, **kwargs): sock.send("{}".format(queued_task.id).encode()) received = sock.recv(1024) sock.close() + return json.loads(received.decode()) + - return received +def query_task(task_id: int) -> models.Task: + return helpers.load_task(task_id) diff --git a/django_leek/migrations/0005_auto_20200305_0753.py b/django_leek/migrations/0005_auto_20200305_0753.py new file mode 100644 index 0000000..ba31800 --- /dev/null +++ b/django_leek/migrations/0005_auto_20200305_0753.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2020-03-05 07:53 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0004_auto_20200304_1134'), + ] + + operations = [ + migrations.RenameField( + model_name='task', + old_name='exception', + new_name='pickled_exception', + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index eac5789..0785cc5 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -1,3 +1,7 @@ +import base64 +import pickle +from typing import Any + from django.db import models @@ -7,5 +11,17 @@ class Task(models.Model): queued_on = models.DateTimeField(auto_now_add=True) started_at = models.DateTimeField(null=True) finished_at = models.DateTimeField(null=True) - exception = models.CharField(max_length=2048, null=True) + pickled_exception = models.CharField(max_length=2048, null=True) pickled_return = models.BinaryField(max_length=4096, null=True) + + @property + def exception(self): + if self.pickled_exception is None: + return None + return pickle.loads(base64.b64decode(self.pickled_exception)) + + @property + def return_value(self): + if self.pickled_return is None: + return None + return pickle.loads(base64.b64decode(self.pickled_return)) diff --git a/django_leek/server.py b/django_leek/server.py index 86b427c..d99ba36 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,4 +1,5 @@ from datetime import datetime +import json import logging import socketserver import multiprocessing @@ -39,7 +40,7 @@ def target(queue): except Exception as e: log.exception("...task failed") task.finished_at = datetime.now() - task.exception = helpers.serialize(e) + task.pickled_exception = helpers.serialize(e) task.save() # workaround to solve problems with django + psycopg2 @@ -66,6 +67,7 @@ def handle(self): # assume a serialized task log.info('Got a task') + response = None try: task_id = int(data.decode()) @@ -84,12 +86,17 @@ def handle(self): self.pools[pool_name].queue.put(task_id) - response = (True, "sent") - self.request.send(str(response).encode()) + response = {'task': 'queued', 'task_id': task_id} except Exception as e: log.exception("failed to queue task") response = (False, "TaskServer Put: {}".format(e).encode(),) - self.request.send(str(response).encode()) + response = { + 'task': 'failed to queue', + 'task_id': task_id, + 'error': str(e) + } + + self.request.send(json.dumps(response).encode()) except OSError as e: # in case of network error, just log diff --git a/test_app/app.py b/test_app/app.py index 7645a2e..cdc4966 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -1,9 +1,14 @@ +import time +import json + from django.conf.urls import url +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder from django.http import HttpResponse from django.shortcuts import render from django.db import models -from django_leek.api import Leek, push_task_to_queue +from django_leek.api import Leek, push_task_to_queue, query_task leek = Leek() @@ -14,28 +19,57 @@ class Person(models.Model): @leek.task def hello(to): + raise ValueError('ops') + #person = Person.objects.create(name="to") + #person.save() + + #print('Hello {}!'.format(to)) + #return 'ok' + + +@leek.task +def slow(seconds: int): person = Person.objects.create(name="to") person.save() - - print('Hello {}!'.format(to)) - return 'wow' + print('sleeping') + time.sleep(seconds) + print('ok') + return 'ok' def index(request): if 'queue' in request.GET: # Run sync - hello(to='sync') + #hello(to='sync') # Run async - hello.offload(to='kwargs') - hello.offload('args') + r = hello.offload(to='kwargs') + #r = slow.offload(seconds=5) push_task_to_queue(hello, to='old') - return render(request, 'index.html', {'message': '✓ task queued'}) + return render(request, 'index.html', { + 'message': '✓ task queued', + 'task_id': r['task_id'] + }) + + return render(request, 'index.html', {'task_id': None}) + - return render(request, 'index.html') +def query(request, task_id): + task = query_task(task_id) + data = { + 'queued_on': task.queued_on, + 'started_at': task.started_at, + 'finished_at': task.finished_at, + 'exception': str(task.exception), + 'return_value': task.return_value + } + return HttpResponse( + json.dumps(data, cls=DjangoJSONEncoder), + content_type='application/json') urlpatterns = [ url(r'^$', index), + url(r'^query/(\d+)/?$', query) ] diff --git a/test_app/templates/index.html b/test_app/templates/index.html index 63eb3ff..749b650 100644 --- a/test_app/templates/index.html +++ b/test_app/templates/index.html @@ -3,8 +3,47 @@

{{ message }}

+

+ \ No newline at end of file From ec0f43511484240468ecbb4092b3664dc20849b5 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Thu, 5 Mar 2020 15:08:21 +0100 Subject: [PATCH 097/109] refactor: consistent naming --- .../migrations/0006_auto_20200305_0804.py | 20 +++++++++++++++++++ django_leek/models.py | 2 +- test_app/app.py | 2 +- test_app/templates/index.html | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 django_leek/migrations/0006_auto_20200305_0804.py diff --git a/django_leek/migrations/0006_auto_20200305_0804.py b/django_leek/migrations/0006_auto_20200305_0804.py new file mode 100644 index 0000000..9c5b108 --- /dev/null +++ b/django_leek/migrations/0006_auto_20200305_0804.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2020-03-05 08:04 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0005_auto_20200305_0753'), + ] + + operations = [ + migrations.RenameField( + model_name='task', + old_name='queued_on', + new_name='queued_at', + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index 0785cc5..d0f458b 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -8,7 +8,7 @@ class Task(models.Model): pickled_task = models.BinaryField(max_length=4096) pool = models.CharField(max_length=256, null=True) - queued_on = models.DateTimeField(auto_now_add=True) + queued_at = models.DateTimeField(auto_now_add=True) started_at = models.DateTimeField(null=True) finished_at = models.DateTimeField(null=True) pickled_exception = models.CharField(max_length=2048, null=True) diff --git a/test_app/app.py b/test_app/app.py index cdc4966..1079d55 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -58,7 +58,7 @@ def index(request): def query(request, task_id): task = query_task(task_id) data = { - 'queued_on': task.queued_on, + 'queued_at': task.queued_at, 'started_at': task.started_at, 'finished_at': task.finished_at, 'exception': str(task.exception), diff --git a/test_app/templates/index.html b/test_app/templates/index.html index 749b650..77f70bc 100644 --- a/test_app/templates/index.html +++ b/test_app/templates/index.html @@ -20,7 +20,7 @@ if (task.started_at !== null) { return "running"; } - if (task.queued_on !== null) { + if (task.queued_at !== null) { return "queued"; } return "pending"; From d87c6d254bfd6d2d3d68dbacc08bf25f8bbe15f0 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Thu, 5 Mar 2020 15:10:42 +0100 Subject: [PATCH 098/109] squashing some migrations --- ..._1134_squashed_0006_auto_20200305_0804.py} | 8 +++++--- .../migrations/0005_auto_20200305_0753.py | 20 ------------------- .../migrations/0006_auto_20200305_0804.py | 20 ------------------- 3 files changed, 5 insertions(+), 43 deletions(-) rename django_leek/migrations/{0004_auto_20200304_1134.py => 0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py} (75%) delete mode 100644 django_leek/migrations/0005_auto_20200305_0753.py delete mode 100644 django_leek/migrations/0006_auto_20200305_0804.py diff --git a/django_leek/migrations/0004_auto_20200304_1134.py b/django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py similarity index 75% rename from django_leek/migrations/0004_auto_20200304_1134.py rename to django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py index 83bb7b6..395e277 100644 --- a/django_leek/migrations/0004_auto_20200304_1134.py +++ b/django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2020-03-04 11:34 +# Generated by Django 1.11 on 2020-03-05 08:06 from __future__ import unicode_literals from django.db import migrations, models @@ -7,6 +7,8 @@ class Migration(migrations.Migration): + replaces = [('django_leek', '0004_auto_20200304_1134'), ('django_leek', '0005_auto_20200305_0753'), ('django_leek', '0006_auto_20200305_0804')] + dependencies = [ ('django_leek', '0003_auto_20180910_1028'), ] @@ -18,10 +20,10 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('pickled_task', models.BinaryField(max_length=4096)), ('pool', models.CharField(max_length=256, null=True)), - ('queued_on', models.DateTimeField(auto_now_add=True)), + ('queued_at', models.DateTimeField(auto_now_add=True)), ('started_at', models.DateTimeField(null=True)), ('finished_at', models.DateTimeField(null=True)), - ('exception', models.CharField(max_length=2048, null=True)), + ('pickled_exception', models.CharField(max_length=2048, null=True)), ('pickled_return', models.BinaryField(max_length=4096, null=True)), ], ), diff --git a/django_leek/migrations/0005_auto_20200305_0753.py b/django_leek/migrations/0005_auto_20200305_0753.py deleted file mode 100644 index ba31800..0000000 --- a/django_leek/migrations/0005_auto_20200305_0753.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2020-03-05 07:53 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_leek', '0004_auto_20200304_1134'), - ] - - operations = [ - migrations.RenameField( - model_name='task', - old_name='exception', - new_name='pickled_exception', - ), - ] diff --git a/django_leek/migrations/0006_auto_20200305_0804.py b/django_leek/migrations/0006_auto_20200305_0804.py deleted file mode 100644 index 9c5b108..0000000 --- a/django_leek/migrations/0006_auto_20200305_0804.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2020-03-05 08:04 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_leek', '0005_auto_20200305_0753'), - ] - - operations = [ - migrations.RenameField( - model_name='task', - old_name='queued_on', - new_name='queued_at', - ), - ] From c4bd9377e1d9a52cdf3d1de647632740f43e3d0d Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Thu, 5 Mar 2020 15:13:32 +0100 Subject: [PATCH 099/109] smarter test app --- test_app/app.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test_app/app.py b/test_app/app.py index 1079d55..c5e1b5c 100644 --- a/test_app/app.py +++ b/test_app/app.py @@ -17,14 +17,18 @@ class Person(models.Model): name = models.CharField(max_length=30) +@leek.task +def fail(): + ValueError('ops') + + @leek.task def hello(to): - raise ValueError('ops') - #person = Person.objects.create(name="to") - #person.save() + person = Person.objects.create(name="to") + person.save() - #print('Hello {}!'.format(to)) - #return 'ok' + print('Hello {}!'.format(to)) + return 'ok' @leek.task @@ -43,8 +47,9 @@ def index(request): #hello(to='sync') # Run async - r = hello.offload(to='kwargs') - #r = slow.offload(seconds=5) + hello.offload(to='kwargs') + fail.offload() + r = slow.offload(seconds=5) push_task_to_queue(hello, to='old') return render(request, 'index.html', { @@ -61,7 +66,7 @@ def query(request, task_id): 'queued_at': task.queued_at, 'started_at': task.started_at, 'finished_at': task.finished_at, - 'exception': str(task.exception), + 'exception': str(task.exception) if task.exception else None, 'return_value': task.return_value } return HttpResponse( From 301ad9a2d6132d96e5c636757c6c540736f88ca9 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Thu, 5 Mar 2020 15:21:52 +0100 Subject: [PATCH 100/109] fixup: test --- django_leek/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django_leek/tests.py b/django_leek/tests.py index 74526b0..2c5351a 100644 --- a/django_leek/tests.py +++ b/django_leek/tests.py @@ -1,3 +1,4 @@ +import json from unittest.mock import patch, MagicMock import socketserver @@ -19,7 +20,7 @@ def test_keyboard_interrupt(self, serve_forever): call_command('leek') -def f(): +def nop(): pass @@ -44,7 +45,8 @@ def test_recv_error(self): self.act() def test_task(self): - task = helpers.save_task_to_db(api.Task(f), 'pool_name') + task = helpers.save_task_to_db(api.Task(nop), 'pool_name') self._request(str(task.id).encode()) self.act() - self.assertEqual(self._response(), b"(True, 'sent')") + actual = json.loads(self._response().decode()) + self.assertEqual(actual, {"task": "queued", "task_id": 1}) From e367fb01aa436d91e596699a4f9ffac8af2bedad Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 10 Mar 2020 14:29:56 +0100 Subject: [PATCH 101/109] timezone aware now() --- django_leek/server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index d99ba36..d21d7d3 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -6,8 +6,10 @@ from .helpers import load_task from . import helpers +from django.utils import timezone import django + log = logging.getLogger(__name__) @@ -29,17 +31,17 @@ def target(queue): task = load_task(task_id=task_id) t = helpers.unpack(task.pickled_task) try: - task.started_at = datetime.now() + task.started_at = timezone.now() task.save() r = t() - task.finished_at = datetime.now() + task.finished_at = timezone.now() task.pickled_return = helpers.serialize(r) task.save() log.info('...successfully') except Exception as e: log.exception("...task failed") - task.finished_at = datetime.now() + task.finished_at = timezone.now() task.pickled_exception = helpers.serialize(e) task.save() From 9dce312fad777b0df7c4f9dbe9a9f4a826ca5a00 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 10 Mar 2020 14:53:24 +0100 Subject: [PATCH 102/109] utility functions for task --- django_leek/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/django_leek/models.py b/django_leek/models.py index d0f458b..000e5e1 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -25,3 +25,12 @@ def return_value(self): if self.pickled_return is None: return None return pickle.loads(base64.b64decode(self.pickled_return)) + + def started(self) -> bool: + return self.started_at is not None + + def finished(self) -> bool: + return self.finished_at is not None + + def successful(self) -> bool: + return self.finished() and self.pickled_return is not None From 3fbc0abe21bb856b5d85fc3e5a5f5c268bb1bfee Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 10 Mar 2020 15:05:59 +0100 Subject: [PATCH 103/109] use BinaryField for pickled data --- .../migrations/0005_auto_20200310_0903.py | 20 +++++++++++++++++++ django_leek/models.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 django_leek/migrations/0005_auto_20200310_0903.py diff --git a/django_leek/migrations/0005_auto_20200310_0903.py b/django_leek/migrations/0005_auto_20200310_0903.py new file mode 100644 index 0000000..0977de6 --- /dev/null +++ b/django_leek/migrations/0005_auto_20200310_0903.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2020-03-10 09:03 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_leek', '0004_auto_20200304_1134_squashed_0006_auto_20200305_0804'), + ] + + operations = [ + migrations.AlterField( + model_name='task', + name='pickled_exception', + field=models.BinaryField(max_length=2048, null=True), + ), + ] diff --git a/django_leek/models.py b/django_leek/models.py index 000e5e1..c0f977c 100644 --- a/django_leek/models.py +++ b/django_leek/models.py @@ -11,7 +11,7 @@ class Task(models.Model): queued_at = models.DateTimeField(auto_now_add=True) started_at = models.DateTimeField(null=True) finished_at = models.DateTimeField(null=True) - pickled_exception = models.CharField(max_length=2048, null=True) + pickled_exception = models.BinaryField(max_length=2048, null=True) pickled_return = models.BinaryField(max_length=4096, null=True) @property From 0a0ca9c6c203fe1fab23cd512c4af4b6a9d8db14 Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 10 Mar 2020 15:19:36 +0100 Subject: [PATCH 104/109] squashing migrations --- ... 0004_new_task_structure_20200310_1518.py} | 6 ++---- .../migrations/0005_auto_20200310_0903.py | 20 ------------------- 2 files changed, 2 insertions(+), 24 deletions(-) rename django_leek/migrations/{0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py => 0004_new_task_structure_20200310_1518.py} (80%) delete mode 100644 django_leek/migrations/0005_auto_20200310_0903.py diff --git a/django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py b/django_leek/migrations/0004_new_task_structure_20200310_1518.py similarity index 80% rename from django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py rename to django_leek/migrations/0004_new_task_structure_20200310_1518.py index 395e277..9bfad2c 100644 --- a/django_leek/migrations/0004_auto_20200304_1134_squashed_0006_auto_20200305_0804.py +++ b/django_leek/migrations/0004_new_task_structure_20200310_1518.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2020-03-05 08:06 +# Generated by Django 1.11 on 2020-03-10 09:18 from __future__ import unicode_literals from django.db import migrations, models @@ -7,8 +7,6 @@ class Migration(migrations.Migration): - replaces = [('django_leek', '0004_auto_20200304_1134'), ('django_leek', '0005_auto_20200305_0753'), ('django_leek', '0006_auto_20200305_0804')] - dependencies = [ ('django_leek', '0003_auto_20180910_1028'), ] @@ -23,7 +21,7 @@ class Migration(migrations.Migration): ('queued_at', models.DateTimeField(auto_now_add=True)), ('started_at', models.DateTimeField(null=True)), ('finished_at', models.DateTimeField(null=True)), - ('pickled_exception', models.CharField(max_length=2048, null=True)), + ('pickled_exception', models.BinaryField(max_length=2048, null=True)), ('pickled_return', models.BinaryField(max_length=4096, null=True)), ], ), diff --git a/django_leek/migrations/0005_auto_20200310_0903.py b/django_leek/migrations/0005_auto_20200310_0903.py deleted file mode 100644 index 0977de6..0000000 --- a/django_leek/migrations/0005_auto_20200310_0903.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2020-03-10 09:03 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('django_leek', '0004_auto_20200304_1134_squashed_0006_auto_20200305_0804'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='pickled_exception', - field=models.BinaryField(max_length=2048, null=True), - ), - ] From 5bb15a7f7bec6e74d883ed8926fd773d4d593f1b Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Tue, 10 Mar 2020 15:30:16 +0100 Subject: [PATCH 105/109] refactor: clearer variable names --- django_leek/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index d21d7d3..3ff554a 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -29,13 +29,13 @@ def target(queue): django.db.connection.close() task = load_task(task_id=task_id) - t = helpers.unpack(task.pickled_task) + pickled_task = helpers.unpack(task.pickled_task) try: task.started_at = timezone.now() task.save() - r = t() + return_value = pickled_task() task.finished_at = timezone.now() - task.pickled_return = helpers.serialize(r) + task.pickled_return = helpers.serialize(return_value) task.save() log.info('...successfully') From 0e9ec496c274d9812cade7b85774ea22748fd4ea Mon Sep 17 00:00:00 2001 From: Samuel Carlsson Date: Wed, 11 Mar 2020 11:41:38 +0100 Subject: [PATCH 106/109] uping version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3c606cb..a15dfbc 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='django-leek', - version='0.9.3', + version='1.0.1', packages=find_packages(exclude=['test_app']), install_requires = ['django>=1.11'], include_package_data=True, From 872e008054879ea216cabbf0d6de34c2612e2608 Mon Sep 17 00:00:00 2001 From: Oskar Lilja Date: Thu, 19 Mar 2020 15:06:16 +0100 Subject: [PATCH 107/109] server.py: when running on osx plaform use threading instead of multiprocessing --- django_leek/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 3ff554a..c3cebc8 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,8 +1,11 @@ from datetime import datetime +from sys import platform import json import logging import socketserver import multiprocessing +import queue +import threading from .helpers import load_task from . import helpers @@ -54,8 +57,13 @@ def target(queue): class Pool(object): def __init__(self): - self.queue = multiprocessing.Queue() - self.worker = multiprocessing.Process(target=target, args=(self.queue,)) + if platform == 'darwin': + # OSX does not support forking + self.queue = queue.Queue() + self.worker = threading.Thread(target=target, args=(self.queue,)) + else: + self.queue = multiprocessing.Queue() + self.worker = multiprocessing.Process(target=target, args=(self.queue,)) class TaskSocketServer(socketserver.BaseRequestHandler): From 18deb82051f4ab5eea05dc7be1838ad6a54456da Mon Sep 17 00:00:00 2001 From: Deniz Akkaya Date: Tue, 30 Jun 2020 17:43:54 +0200 Subject: [PATCH 108/109] fixing multiple jobs in a task --- django_leek/server.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index c3cebc8..4886161 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -1,5 +1,6 @@ from datetime import datetime from sys import platform +from queue import Empty import json import logging import socketserver @@ -21,8 +22,9 @@ def target(queue): log.info('Worker Starts') done = False while not done: - task_id = queue.get() - if task_id is None: + try: + task_id = queue.get(block=True,timeout=1) + except Empty as e: done = True break @@ -42,6 +44,7 @@ def target(queue): task.save() log.info('...successfully') + queue.task_done() except Exception as e: log.exception("...task failed") task.finished_at = timezone.now() @@ -59,10 +62,10 @@ class Pool(object): def __init__(self): if platform == 'darwin': # OSX does not support forking - self.queue = queue.Queue() + self.queue = queue.Queue(maxsize=10000) self.worker = threading.Thread(target=target, args=(self.queue,)) else: - self.queue = multiprocessing.Queue() + self.queue = multiprocessing.Queue(maxsize=10000) self.worker = multiprocessing.Process(target=target, args=(self.queue,)) @@ -80,11 +83,11 @@ def handle(self): response = None try: task_id = int(data.decode()) - + # Connection are closed by tasks, force it to reconnect django.db.connections.close_all() task = load_task(task_id=task_id) - + # Ensure pool got a worker processing it pool_name = task.pool or self.DEFAULT_POOL pool = self.pools.get(pool_name) @@ -105,7 +108,7 @@ def handle(self): 'task_id': task_id, 'error': str(e) } - + self.request.send(json.dumps(response).encode()) except OSError as e: @@ -113,5 +116,4 @@ def handle(self): log.exception("network error") def finish(self): - for pool in self.pools.values(): - pool.queue.put(None) + None From 01199004466b291b8328700d1c3eaa76eacbe1d8 Mon Sep 17 00:00:00 2001 From: Deniz Akkaya Date: Wed, 1 Jul 2020 10:03:12 +0200 Subject: [PATCH 109/109] moving max queue size to defined variable --- django_leek/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/django_leek/server.py b/django_leek/server.py index 4886161..cc49416 100644 --- a/django_leek/server.py +++ b/django_leek/server.py @@ -16,6 +16,7 @@ log = logging.getLogger(__name__) +MAX_QUEUE_SIZE = 10000 def target(queue): django.setup() @@ -23,7 +24,7 @@ def target(queue): done = False while not done: try: - task_id = queue.get(block=True,timeout=1) + task_id = queue.get(block=True, timeout=1) except Empty as e: done = True break @@ -62,10 +63,10 @@ class Pool(object): def __init__(self): if platform == 'darwin': # OSX does not support forking - self.queue = queue.Queue(maxsize=10000) + self.queue = queue.Queue(maxsize=MAX_QUEUE_SIZE) self.worker = threading.Thread(target=target, args=(self.queue,)) else: - self.queue = multiprocessing.Queue(maxsize=10000) + self.queue = multiprocessing.Queue(maxsize=MAX_QUEUE_SIZE) self.worker = multiprocessing.Process(target=target, args=(self.queue,))