From 52d5232348d7dfdb2b05be1ce3aa4423c44b01bd Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 6 May 2015 18:54:41 +0100 Subject: [PATCH 001/118] I do everything flake8 tells me --- src/asynqp/__init__.py | 12 ++++++------ src/asynqp/bases.py | 6 ------ src/asynqp/exchange.py | 1 - src/asynqp/protocol.py | 7 +++---- test/base_contexts.py | 8 ++++---- test/channel_tests.py | 1 - test/connection_tests.py | 2 +- test/exchange_tests.py | 1 - test/method_tests.py | 2 -- test/queue_tests.py | 1 - test/util.py | 3 ++- 11 files changed, 16 insertions(+), 28 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index d649527..8577051 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,10 +1,10 @@ import asyncio -from .exceptions import AMQPError, Deleted -from .message import Message, IncomingMessage -from .connection import Connection -from .channel import Channel -from .exchange import Exchange -from .queue import Queue, QueueBinding, Consumer +from .exceptions import AMQPError, Deleted # noqa +from .message import Message, IncomingMessage # noqa +from .connection import Connection # noqa +from .channel import Channel # noqa +from .exchange import Exchange # noqa +from .queue import Queue, QueueBinding, Consumer # noqa @asyncio.coroutine diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py index e9922c1..0cb0ff3 100644 --- a/src/asynqp/bases.py +++ b/src/asynqp/bases.py @@ -1,9 +1,3 @@ -import asyncio -from . import spec -from . import frames -from .exceptions import AMQPError - - class Sender(object): def __init__(self, channel_id, protocol): self.channel_id = channel_id diff --git a/src/asynqp/exchange.py b/src/asynqp/exchange.py index 78461b2..531c7e7 100644 --- a/src/asynqp/exchange.py +++ b/src/asynqp/exchange.py @@ -54,4 +54,3 @@ def delete(self, *, if_unused=True): self.sender.send_ExchangeDelete(self.name, if_unused) yield from self.synchroniser.await(spec.ExchangeDeleteOK) self.reader.ready() - diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index b7b1cb5..7fcf918 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -2,7 +2,6 @@ import struct from . import spec from . import frames -from .routing import Dispatcher from .exceptions import AMQPError @@ -68,14 +67,14 @@ def read_frame(self, data): self.partial_frame = data return - raw_payload = data[7:7+size] - frame_end = data[7+size] + raw_payload = data[7:7 + size] + frame_end = data[7 + size] if frame_end != spec.FRAME_END: raise AMQPError("Frame end byte was incorrect") frame = frames.read(frame_type, channel_id, raw_payload) - remainder = data[8+size:] + remainder = data[8 + size:] return frame, remainder diff --git a/test/base_contexts.py b/test/base_contexts.py index 12709ba..923c3a5 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -33,7 +33,7 @@ def async_partial(self, coro): class MockServerContext(LoopContext): def given_a_mock_server_on_the_other_end_of_the_transport(self): - self.dispatcher = protocol.Dispatcher() + self.dispatcher = asynqp.routing.Dispatcher() self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.server = MockServer(self.protocol, self.tick) self.transport = FakeTransport(self.server) @@ -120,7 +120,7 @@ def given_the_pieces_i_need_for_a_connection(self): self.protocol.transport = mock.Mock() self.protocol.send_frame._is_coroutine = False # :( - self.dispatcher = protocol.Dispatcher() + self.dispatcher = asynqp.routing.Dispatcher() self.connection_info = ConnectionInfo('guest', 'guest', '/') @@ -149,7 +149,7 @@ def given_an_open_connection(self): class ProtocolContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) - self.dispatcher = protocol.Dispatcher() + self.dispatcher = asynqp.routing.Dispatcher() self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.protocol.connection_made(self.transport) @@ -157,6 +157,6 @@ def given_a_connected_protocol(self): class MockDispatcherContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) - self.dispatcher = mock.Mock(spec=protocol.Dispatcher) + self.dispatcher = mock.Mock(spec=asynqp.routing.Dispatcher) self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.protocol.connection_made(self.transport) diff --git a/test/channel_tests.py b/test/channel_tests.py index b260d33..8447df3 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -209,7 +209,6 @@ def return_msg(self): self.tick() - class WhenTheHandlerIsNotCallable(OpenChannelContext): def when_I_set_the_handler(self): self.exception = contexts.catch(self.channel.set_return_handler, "i am not callable") diff --git a/test/connection_tests.py b/test/connection_tests.py index 75b3d76..a55b785 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -2,7 +2,7 @@ import sys import asynqp from unittest import mock -from asynqp import spec, protocol, frames +from asynqp import spec from asynqp.connection import open_connection, ConnectionInfo from .base_contexts import LegacyOpenConnectionContext, MockServerContext, OpenConnectionContext diff --git a/test/exchange_tests.py b/test/exchange_tests.py index 3409725..72ca7db 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -1,7 +1,6 @@ import asyncio import uuid from datetime import datetime -from unittest import mock import asynqp from asynqp import spec from asynqp import frames diff --git a/test/method_tests.py b/test/method_tests.py index 79bc83e..5c0fc23 100644 --- a/test/method_tests.py +++ b/test/method_tests.py @@ -1,5 +1,4 @@ import asynqp -from unittest import mock from asynqp import spec from asynqp import frames from asynqp import amqptypes @@ -165,4 +164,3 @@ def when_the_frame_arrives(self): def it_should_deserialise_it_to_the_correct_method(self): self.dispatcher.dispatch.assert_called_once_with(self.expected_frame) - diff --git a/test/queue_tests.py b/test/queue_tests.py index b2753c3..06d5bda 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -1,6 +1,5 @@ import asyncio from datetime import datetime -from unittest import mock import asynqp from asynqp import message from asynqp import frames diff --git a/test/util.py b/test/util.py index bfea180..185395d 100644 --- a/test/util.py +++ b/test/util.py @@ -1,4 +1,5 @@ import asyncio +from contextlib import contextmanager from unittest import mock import asynqp.frames from asynqp import protocol @@ -94,10 +95,10 @@ def __eq__(self, other): return _any() -from contextlib import contextmanager @contextmanager def silence_expected_destroy_pending_log(expected_coro_name=''): real_async = asyncio.async + def async(*args, **kwargs): t = real_async(*args, **kwargs) if expected_coro_name in repr(t): From ab3de4aaaec6d2af4d4f1198b30c04d3f81d8c8f Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 6 May 2015 18:56:32 +0100 Subject: [PATCH 002/118] add gitattributes and fix up line endings --- .gitattributes | 2 + .gitignore | 112 +- RELEASING | 16 +- doc/make.bat | 484 ++-- src/asynqp/amqp0-9-1.xml | 5686 +++++++++++++++++++------------------- src/asynqp/bases.py | 42 +- src/asynqp/routing.py | 252 +- test/util.py | 218 +- 8 files changed, 3407 insertions(+), 3405 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/.gitignore b/.gitignore index 8875eb4..345208f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,56 +1,56 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -bin/ -build/ -develop-eggs/ -dist/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# Rope -.ropeproject - -# Django stuff: -*.log -*.pot - -# Sphinx documentation -doc/_build/ - -*.sublime-workspace -.ipynb_checkpoints +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +doc/_build/ + +*.sublime-workspace +.ipynb_checkpoints diff --git a/RELEASING b/RELEASING index d15d97d..6163b2b 100644 --- a/RELEASING +++ b/RELEASING @@ -1,8 +1,8 @@ -1. Check Travis is green on the latest commit -2. Increment the version in setup.py -3. Increment the version in doc/conf.py -4. Commit and tag -5. Push using --follow-tags -6. Check that the tests passed on Travis and it published to the Cheese Shop -6a. If it failed to publish to the cheese shop for some reason, run `python setup.py register` followed by `python setup.py sdist bdist_egg bdist_wheel upload` -7. Build the docs at readthedocs.org and increment the latest version +1. Check Travis is green on the latest commit +2. Increment the version in setup.py +3. Increment the version in doc/conf.py +4. Commit and tag +5. Push using --follow-tags +6. Check that the tests passed on Travis and it published to the Cheese Shop +6a. If it failed to publish to the cheese shop for some reason, run `python setup.py register` followed by `python setup.py sdist bdist_egg bdist_wheel upload` +7. Build the docs at readthedocs.org and increment the latest version diff --git a/doc/make.bat b/doc/make.bat index bc72244..388cf2c 100644 --- a/doc/make.bat +++ b/doc/make.bat @@ -1,242 +1,242 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\asynqp.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\asynqp.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\asynqp.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\asynqp.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/src/asynqp/amqp0-9-1.xml b/src/asynqp/amqp0-9-1.xml index 5ddab06..da785eb 100644 --- a/src/asynqp/amqp0-9-1.xml +++ b/src/asynqp/amqp0-9-1.xml @@ -1,2843 +1,2843 @@ - - - - - - - - - - - - - - - - - - - - - - - Indicates that the method completed successfully. This reply code is - reserved for future use - the current protocol design does not use positive - confirmation and reply codes are sent only in case of an error. - - - - - - The client attempted to transfer content larger than the server could accept - at the present time. The client may retry at a later time. - - - - - - When the exchange cannot deliver to a consumer when the immediate flag is - set. As a result of pending data on the queue or the absence of any - consumers of the queue. - - - - - - An operator intervened to close the connection for some reason. The client - may retry at some later date. - - - - - - The client tried to work with an unknown virtual host. - - - - - - The client attempted to work with a server entity to which it has no - access due to security settings. - - - - - - The client attempted to work with a server entity that does not exist. - - - - - - The client attempted to work with a server entity to which it has no - access because another client is working with it. - - - - - - The client requested a method that was not allowed because some precondition - failed. - - - - - - The sender sent a malformed frame that the recipient could not decode. - This strongly implies a programming error in the sending peer. - - - - - - The sender sent a frame that contained illegal values for one or more - fields. This strongly implies a programming error in the sending peer. - - - - - - The client sent an invalid sequence of frames, attempting to perform an - operation that was considered invalid by the server. This usually implies - a programming error in the client. - - - - - - The client attempted to work with a channel that had not been correctly - opened. This most likely indicates a fault in the client layer. - - - - - - The peer sent a frame that was not expected, usually in the context of - a content header and body. This strongly indicates a fault in the peer's - content processing. - - - - - - The server could not complete the method because it lacked sufficient - resources. This may be due to the client creating too many of some type - of entity. - - - - - - The client tried to work with some entity in a manner that is prohibited - by the server, due to security settings or by some other criteria. - - - - - - The client tried to use functionality that is not implemented in the - server. - - - - - - The server could not complete the method because of an internal error. - The server may require intervention by an operator in order to resume - normal operations. - - - - - - - - - - Identifier for the consumer, valid within the current channel. - - - - - - The server-assigned and channel-specific delivery tag - - - - The delivery tag is valid only within the channel from which the message was - received. I.e. a client MUST NOT receive a message on one channel and then - acknowledge it on another. - - - - - The server MUST NOT use a zero value for delivery tags. Zero is reserved - for client use, meaning "all messages so far received". - - - - - - - The exchange name is a client-selected string that identifies the exchange for - publish methods. - - - - - - - - - - If this field is set the server does not expect acknowledgements for - messages. That is, when a message is delivered to the client the server - assumes the delivery will succeed and immediately dequeues it. This - functionality may increase performance but at the cost of reliability. - Messages can get lost if a client dies before they are delivered to the - application. - - - - - - If the no-local field is set the server will not send messages to the connection that - published them. - - - - - - If set, the server will not respond to the method. The client should not wait - for a reply method. If the server could not complete the method it will raise a - channel or connection exception. - - - - - - Unconstrained. - - - - - - - - This table provides a set of peer properties, used for identification, debugging, - and general information. - - - - - - The queue name identifies the queue within the vhost. In methods where the queue - name may be blank, and that has no specific significance, this refers to the - 'current' queue for the channel, meaning the last queue that the client declared - on the channel. If the client did not declare a queue, and the method needs a - queue name, this will result in a 502 (syntax error) channel exception. - - - - - - - - This indicates that the message has been previously delivered to this or - another client. - - - - The server SHOULD try to signal redelivered messages when it can. When - redelivering a message that was not successfully acknowledged, the server - SHOULD deliver it to the original client if possible. - - - Declare a shared queue and publish a message to the queue. Consume the - message using explicit acknowledgements, but do not acknowledge the - message. Close the connection, reconnect, and consume from the queue - again. The message should arrive with the redelivered flag set. - - - - - The client MUST NOT rely on the redelivered field but should take it as a - hint that the message may already have been processed. A fully robust - client must be able to track duplicate received messages on non-transacted, - and locally-transacted channels. - - - - - - - The number of messages in the queue, which will be zero for newly-declared - queues. This is the number of messages present in the queue, and committed - if the channel on which they were published is transacted, that are not - waiting acknowledgement. - - - - - - The reply code. The AMQ reply codes are defined as constants at the start - of this formal specification. - - - - - - - The localised reply text. This text can be logged as an aid to resolving - issues. - - - - - - - - - - - - - - - - - - - - The connection class provides methods for a client to establish a network connection to - a server, and for both peers to operate the connection thereafter. - - - - connection = open-connection *use-connection close-connection - open-connection = C:protocol-header - S:START C:START-OK - *challenge - S:TUNE C:TUNE-OK - C:OPEN S:OPEN-OK - challenge = S:SECURE C:SECURE-OK - use-connection = *channel - close-connection = C:CLOSE S:CLOSE-OK - / S:CLOSE C:CLOSE-OK - - - - - - - - - - This method starts the connection negotiation process by telling the client the - protocol version that the server proposes, along with a list of security mechanisms - which the client can use for authentication. - - - - - If the server cannot support the protocol specified in the protocol header, - it MUST respond with a valid protocol header and then close the socket - connection. - - - The client sends a protocol header containing an invalid protocol name. - The server MUST respond by sending a valid protocol header and then closing - the connection. - - - - - The server MUST provide a protocol version that is lower than or equal to - that requested by the client in the protocol header. - - - The client requests a protocol version that is higher than any valid - implementation, e.g. 2.0. The server must respond with a protocol header - indicating its supported protocol version, e.g. 1.0. - - - - - If the client cannot handle the protocol version suggested by the server - it MUST close the socket connection without sending any further data. - - - The server sends a protocol version that is lower than any valid - implementation, e.g. 0.1. The client must respond by closing the - connection without sending any further data. - - - - - - - - - The major version number can take any value from 0 to 99 as defined in the - AMQP specification. - - - - - - The minor version number can take any value from 0 to 99 as defined in the - AMQP specification. - - - - - - - The properties SHOULD contain at least these fields: "host", specifying the - server host name or address, "product", giving the name of the server product, - "version", giving the name of the server version, "platform", giving the name - of the operating system, "copyright", if appropriate, and "information", giving - other general information. - - - Client connects to server and inspects the server properties. It checks for - the presence of the required fields. - - - - - - - A list of the security mechanisms that the server supports, delimited by spaces. - - - - - - - A list of the message locales that the server supports, delimited by spaces. The - locale defines the language in which the server will send reply texts. - - - - The server MUST support at least the en_US locale. - - - Client connects to server and inspects the locales field. It checks for - the presence of the required locale(s). - - - - - - - - - This method selects a SASL security mechanism. - - - - - - - - - The properties SHOULD contain at least these fields: "product", giving the name - of the client product, "version", giving the name of the client version, "platform", - giving the name of the operating system, "copyright", if appropriate, and - "information", giving other general information. - - - - - - - A single security mechanisms selected by the client, which must be one of those - specified by the server. - - - - The client SHOULD authenticate using the highest-level security profile it - can handle from the list provided by the server. - - - - - If the mechanism field does not contain one of the security mechanisms - proposed by the server in the Start method, the server MUST close the - connection without sending any further data. - - - Client connects to server and sends an invalid security mechanism. The - server must respond by closing the connection (a socket close, with no - connection close negotiation). - - - - - - - - A block of opaque data passed to the security mechanism. The contents of this - data are defined by the SASL security mechanism. - - - - - - - A single message locale selected by the client, which must be one of those - specified by the server. - - - - - - - - - - The SASL protocol works by exchanging challenges and responses until both peers have - received sufficient information to authenticate each other. This method challenges - the client to provide more information. - - - - - - - - Challenge information, a block of opaque binary data passed to the security - mechanism. - - - - - - - This method attempts to authenticate, passing a block of SASL data for the security - mechanism at the server side. - - - - - - - A block of opaque data passed to the security mechanism. The contents of this - data are defined by the SASL security mechanism. - - - - - - - - - - This method proposes a set of connection configuration values to the client. The - client can accept and/or adjust these. - - - - - - - - - Specifies highest channel number that the server permits. Usable channel numbers - are in the range 1..channel-max. Zero indicates no specified limit. - - - - - - The largest frame size that the server proposes for the connection, including - frame header and end-byte. The client can negotiate a lower value. Zero means - that the server does not impose any specific limit but may reject very large - frames if it cannot allocate resources for them. - - - - Until the frame-max has been negotiated, both peers MUST accept frames of up - to frame-min-size octets large, and the minimum negotiated value for frame-max - is also frame-min-size. - - - Client connects to server and sends a large properties field, creating a frame - of frame-min-size octets. The server must accept this frame. - - - - - - - The delay, in seconds, of the connection heartbeat that the server wants. - Zero means the server does not want a heartbeat. - - - - - - - This method sends the client's connection tuning parameters to the server. - Certain fields are negotiated, others provide capability information. - - - - - - - The maximum total number of channels that the client will use per connection. - - - - If the client specifies a channel max that is higher than the value provided - by the server, the server MUST close the connection without attempting a - negotiated close. The server may report the error in some fashion to assist - implementors. - - - - - - - - - The largest frame size that the client and server will use for the connection. - Zero means that the client does not impose any specific limit but may reject - very large frames if it cannot allocate resources for them. Note that the - frame-max limit applies principally to content frames, where large contents can - be broken into frames of arbitrary size. - - - - Until the frame-max has been negotiated, both peers MUST accept frames of up - to frame-min-size octets large, and the minimum negotiated value for frame-max - is also frame-min-size. - - - - - If the client specifies a frame max that is higher than the value provided - by the server, the server MUST close the connection without attempting a - negotiated close. The server may report the error in some fashion to assist - implementors. - - - - - - - The delay, in seconds, of the connection heartbeat that the client wants. Zero - means the client does not want a heartbeat. - - - - - - - - - This method opens a connection to a virtual host, which is a collection of - resources, and acts to separate multiple application domains within a server. - The server may apply arbitrary limits per virtual host, such as the number - of each type of entity that may be used, per connection and/or in total. - - - - - - - - The name of the virtual host to work with. - - - - If the server supports multiple virtual hosts, it MUST enforce a full - separation of exchanges, queues, and all associated entities per virtual - host. An application, connected to a specific virtual host, MUST NOT be able - to access resources of another virtual host. - - - - - The server SHOULD verify that the client has permission to access the - specified virtual host. - - - - - - - - - - - - This method signals to the client that the connection is ready for use. - - - - - - - - - - - This method indicates that the sender wants to close the connection. This may be - due to internal conditions (e.g. a forced shut-down) or due to an error handling - a specific method, i.e. an exception. When a close is due to an exception, the - sender provides the class and method id of the method which caused the exception. - - - - After sending this method, any received methods except Close and Close-OK MUST - be discarded. The response to receiving a Close after sending Close must be to - send Close-Ok. - - - - - - - - - - - - - When the close is provoked by a method exception, this is the class of the - method. - - - - - - When the close is provoked by a method exception, this is the ID of the method. - - - - - - - This method confirms a Connection.Close method and tells the recipient that it is - safe to release resources for the connection and close the socket. - - - - A peer that detects a socket closure without having received a Close-Ok - handshake method SHOULD log the error. - - - - - - - - - - - - The channel class provides methods for a client to establish a channel to a - server and for both peers to operate the channel thereafter. - - - - channel = open-channel *use-channel close-channel - open-channel = C:OPEN S:OPEN-OK - use-channel = C:FLOW S:FLOW-OK - / S:FLOW C:FLOW-OK - / functional-class - close-channel = C:CLOSE S:CLOSE-OK - / S:CLOSE C:CLOSE-OK - - - - - - - - - - This method opens a channel to the server. - - - - The client MUST NOT use this method on an already-opened channel. - - - Client opens a channel and then reopens the same channel. - - - - - - - - - - - This method signals to the client that the channel is ready for use. - - - - - - - - - - - This method asks the peer to pause or restart the flow of content data sent by - a consumer. This is a simple flow-control mechanism that a peer can use to avoid - overflowing its queues or otherwise finding itself receiving more messages than - it can process. Note that this method is not intended for window control. It does - not affect contents returned by Basic.Get-Ok methods. - - - - - When a new channel is opened, it is active (flow is active). Some applications - assume that channels are inactive until started. To emulate this behaviour a - client MAY open the channel, then pause it. - - - - - - When sending content frames, a peer SHOULD monitor the channel for incoming - methods and respond to a Channel.Flow as rapidly as possible. - - - - - - A peer MAY use the Channel.Flow method to throttle incoming content data for - internal reasons, for example, when exchanging data over a slower connection. - - - - - - The peer that requests a Channel.Flow method MAY disconnect and/or ban a peer - that does not respect the request. This is to prevent badly-behaved clients - from overwhelming a server. - - - - - - - - - - - If 1, the peer starts sending content frames. If 0, the peer stops sending - content frames. - - - - - - - Confirms to the peer that a flow command was received and processed. - - - - - - Confirms the setting of the processed flow method: 1 means the peer will start - sending or continue to send content frames; 0 means it will not. - - - - - - - - - This method indicates that the sender wants to close the channel. This may be due to - internal conditions (e.g. a forced shut-down) or due to an error handling a specific - method, i.e. an exception. When a close is due to an exception, the sender provides - the class and method id of the method which caused the exception. - - - - After sending this method, any received methods except Close and Close-OK MUST - be discarded. The response to receiving a Close after sending Close must be to - send Close-Ok. - - - - - - - - - - - - - When the close is provoked by a method exception, this is the class of the - method. - - - - - - When the close is provoked by a method exception, this is the ID of the method. - - - - - - - This method confirms a Channel.Close method and tells the recipient that it is safe - to release resources for the channel. - - - - A peer that detects a socket closure without having received a Channel.Close-Ok - handshake method SHOULD log the error. - - - - - - - - - - - - Exchanges match and distribute messages across queues. Exchanges can be configured in - the server or declared at runtime. - - - - exchange = C:DECLARE S:DECLARE-OK - / C:DELETE S:DELETE-OK - - - - - - - - The server MUST implement these standard exchange types: fanout, direct. - - - Client attempts to declare an exchange with each of these standard types. - - - - - The server SHOULD implement these standard exchange types: topic, headers. - - - Client attempts to declare an exchange with each of these standard types. - - - - - The server MUST, in each virtual host, pre-declare an exchange instance - for each standard exchange type that it implements, where the name of the - exchange instance, if defined, is "amq." followed by the exchange type name. - - - The server MUST, in each virtual host, pre-declare at least two direct - exchange instances: one named "amq.direct", the other with no public name - that serves as a default exchange for Publish methods. - - - Client declares a temporary queue and attempts to bind to each required - exchange instance ("amq.fanout", "amq.direct", "amq.topic", and "amq.headers" - if those types are defined). - - - - - The server MUST pre-declare a direct exchange with no public name to act as - the default exchange for content Publish methods and for default queue bindings. - - - Client checks that the default exchange is active by specifying a queue - binding with no exchange name, and publishing a message with a suitable - routing key but without specifying the exchange name, then ensuring that - the message arrives in the queue correctly. - - - - - The server MUST NOT allow clients to access the default exchange except - by specifying an empty exchange name in the Queue.Bind and content Publish - methods. - - - - - The server MAY implement other exchange types as wanted. - - - - - - - - This method creates an exchange if it does not already exist, and if the exchange - exists, verifies that it is of the correct and expected class. - - - - The server SHOULD support a minimum of 16 exchanges per virtual host and - ideally, impose no limit except as defined by available resources. - - - The client declares as many exchanges as it can until the server reports - an error; the number of exchanges successfully declared must be at least - sixteen. - - - - - - - - - - - - - Exchange names starting with "amq." are reserved for pre-declared and - standardised exchanges. The client MAY declare an exchange starting with - "amq." if the passive option is set, or the exchange already exists. - - - The client attempts to declare a non-existing exchange starting with - "amq." and with the passive option set to zero. - - - - - The exchange name consists of a non-empty sequence of these characters: - letters, digits, hyphen, underscore, period, or colon. - - - The client attempts to declare an exchange with an illegal name. - - - - - - - - Each exchange belongs to one of a set of exchange types implemented by the - server. The exchange types define the functionality of the exchange - i.e. how - messages are routed through it. It is not valid or meaningful to attempt to - change the type of an existing exchange. - - - - Exchanges cannot be redeclared with different types. The client MUST not - attempt to redeclare an existing exchange with a different type than used - in the original Exchange.Declare method. - - - TODO. - - - - - The client MUST NOT attempt to declare an exchange with a type that the - server does not support. - - - TODO. - - - - - - - If set, the server will reply with Declare-Ok if the exchange already - exists with the same name, and raise an error if not. The client can - use this to check whether an exchange exists without modifying the - server state. When set, all other method fields except name and no-wait - are ignored. A declare with both passive and no-wait has no effect. - Arguments are compared for semantic equivalence. - - - - If set, and the exchange does not already exist, the server MUST - raise a channel exception with reply code 404 (not found). - - - TODO. - - - - - If not set and the exchange exists, the server MUST check that the - existing exchange has the same values for type, durable, and arguments - fields. The server MUST respond with Declare-Ok if the requested - exchange matches these fields, and MUST raise a channel exception if - not. - - - TODO. - - - - - - - If set when creating a new exchange, the exchange will be marked as durable. - Durable exchanges remain active when a server restarts. Non-durable exchanges - (transient exchanges) are purged if/when a server restarts. - - - - The server MUST support both durable and transient exchanges. - - - TODO. - - - - - - - - - - - - - A set of arguments for the declaration. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - This method confirms a Declare method and confirms the name of the exchange, - essential for automatically-named exchanges. - - - - - - - - - This method deletes an exchange. When an exchange is deleted all queue bindings on - the exchange are cancelled. - - - - - - - - - - - - The client MUST NOT attempt to delete an exchange that does not exist. - - - - - - - - If set, the server will only delete the exchange if it has no queue bindings. If - the exchange has queue bindings the server does not delete it but raises a - channel exception instead. - - - - The server MUST NOT delete an exchange that has bindings on it, if the if-unused - field is true. - - - The client declares an exchange, binds a queue to it, then tries to delete it - setting if-unused to true. - - - - - - - - - This method confirms the deletion of an exchange. - - - - - - - - - Queues store and forward messages. Queues can be configured in the server or created at - runtime. Queues must be attached to at least one exchange in order to receive messages - from publishers. - - - - queue = C:DECLARE S:DECLARE-OK - / C:BIND S:BIND-OK - / C:UNBIND S:UNBIND-OK - / C:PURGE S:PURGE-OK - / C:DELETE S:DELETE-OK - - - - - - - - - - This method creates or checks a queue. When creating a new queue the client can - specify various properties that control the durability of the queue and its - contents, and the level of sharing for the queue. - - - - - The server MUST create a default binding for a newly-declared queue to the - default exchange, which is an exchange of type 'direct' and use the queue - name as the routing key. - - - Client declares a new queue, and then without explicitly binding it to an - exchange, attempts to send a message through the default exchange binding, - i.e. publish a message to the empty exchange, with the queue name as routing - key. - - - - - - The server SHOULD support a minimum of 256 queues per virtual host and ideally, - impose no limit except as defined by available resources. - - - Client attempts to declare as many queues as it can until the server reports - an error. The resulting count must at least be 256. - - - - - - - - - - - - - The queue name MAY be empty, in which case the server MUST create a new - queue with a unique generated name and return this to the client in the - Declare-Ok method. - - - Client attempts to declare several queues with an empty name. The client then - verifies that the server-assigned names are unique and different. - - - - - Queue names starting with "amq." are reserved for pre-declared and - standardised queues. The client MAY declare a queue starting with - "amq." if the passive option is set, or the queue already exists. - - - The client attempts to declare a non-existing queue starting with - "amq." and with the passive option set to zero. - - - - - The queue name can be empty, or a sequence of these characters: - letters, digits, hyphen, underscore, period, or colon. - - - The client attempts to declare a queue with an illegal name. - - - - - - - If set, the server will reply with Declare-Ok if the queue already - exists with the same name, and raise an error if not. The client can - use this to check whether a queue exists without modifying the - server state. When set, all other method fields except name and no-wait - are ignored. A declare with both passive and no-wait has no effect. - Arguments are compared for semantic equivalence. - - - - The client MAY ask the server to assert that a queue exists without - creating the queue if not. If the queue does not exist, the server - treats this as a failure. - - - Client declares an existing queue with the passive option and expects - the server to respond with a declare-ok. Client then attempts to declare - a non-existent queue with the passive option, and the server must close - the channel with the correct reply-code. - - - - - If not set and the queue exists, the server MUST check that the - existing queue has the same values for durable, exclusive, auto-delete, - and arguments fields. The server MUST respond with Declare-Ok if the - requested queue matches these fields, and MUST raise a channel exception - if not. - - - TODO. - - - - - - - If set when creating a new queue, the queue will be marked as durable. Durable - queues remain active when a server restarts. Non-durable queues (transient - queues) are purged if/when a server restarts. Note that durable queues do not - necessarily hold persistent messages, although it does not make sense to send - persistent messages to a transient queue. - - - - The server MUST recreate the durable queue after a restart. - - - Client declares a durable queue. The server is then restarted. The client - then attempts to send a message to the queue. The message should be successfully - delivered. - - - - - The server MUST support both durable and transient queues. - - A client declares two named queues, one durable and one transient. - - - - - - - Exclusive queues may only be accessed by the current connection, and are - deleted when that connection closes. Passive declaration of an exclusive - queue by other connections are not allowed. - - - - - The server MUST support both exclusive (private) and non-exclusive (shared) - queues. - - - A client declares two named queues, one exclusive and one non-exclusive. - - - - - - The client MAY NOT attempt to use a queue that was declared as exclusive - by another still-open connection. - - - One client declares an exclusive queue. A second client on a different - connection attempts to declare, bind, consume, purge, delete, or declare - a queue of the same name. - - - - - - - If set, the queue is deleted when all consumers have finished using it. The last - consumer can be cancelled either explicitly or because its channel is closed. If - there was no consumer ever on the queue, it won't be deleted. Applications can - explicitly delete auto-delete queues using the Delete method as normal. - - - - - The server MUST ignore the auto-delete field if the queue already exists. - - - Client declares two named queues, one as auto-delete and one explicit-delete. - Client then attempts to declare the two queues using the same names again, - but reversing the value of the auto-delete field in each case. Verify that the - queues still exist with the original auto-delete flag values. - - - - - - - - - A set of arguments for the declaration. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - This method confirms a Declare method and confirms the name of the queue, essential - for automatically-named queues. - - - - - - - Reports the name of the queue. If the server generated a queue name, this field - contains that name. - - - - - - - - - Reports the number of active consumers for the queue. Note that consumers can - suspend activity (Channel.Flow) in which case they do not appear in this count. - - - - - - - - - This method binds a queue to an exchange. Until a queue is bound it will not - receive any messages. In a classic messaging model, store-and-forward queues - are bound to a direct exchange and subscription queues are bound to a topic - exchange. - - - - - A server MUST allow ignore duplicate bindings - that is, two or more bind - methods for a specific queue, with identical arguments - without treating these - as an error. - - - A client binds a named queue to an exchange. The client then repeats the bind - (with identical arguments). - - - - - - A server MUST not deliver the same message more than once to a queue, even if - the queue has multiple bindings that match the message. - - - A client declares a named queue and binds it using multiple bindings to the - amq.topic exchange. The client then publishes a message that matches all its - bindings. - - - - - - The server MUST allow a durable queue to bind to a transient exchange. - - - A client declares a transient exchange. The client then declares a named durable - queue and then attempts to bind the transient exchange to the durable queue. - - - - - - Bindings of durable queues to durable exchanges are automatically durable - and the server MUST restore such bindings after a server restart. - - - A server declares a named durable queue and binds it to a durable exchange. The - server is restarted. The client then attempts to use the queue/exchange combination. - - - - - - The server SHOULD support at least 4 bindings per queue, and ideally, impose no - limit except as defined by available resources. - - - A client declares a named queue and attempts to bind it to 4 different - exchanges. - - - - - - - - - - - - Specifies the name of the queue to bind. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to bind an unnamed queue. - - - - - The client MUST NOT attempt to bind a queue that does not exist. - - - The client attempts to bind a non-existent queue. - - - - - - - - A client MUST NOT be allowed to bind a queue to a non-existent exchange. - - - A client attempts to bind an named queue to a undeclared exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - - - Specifies the routing key for the binding. The routing key is used for routing - messages depending on the exchange configuration. Not all exchanges use a - routing key - refer to the specific exchange documentation. If the queue name - is empty, the server uses the last queue declared on the channel. If the - routing key is also empty, the server uses this queue name for the routing - key as well. If the queue name is provided but the routing key is empty, the - server does the binding with that empty routing key. The meaning of empty - routing keys depends on the exchange implementation. - - - - If a message queue binds to a direct exchange using routing key K and a - publisher sends the exchange a message with routing key R, then the message - MUST be passed to the message queue if K = R. - - - - - - - - - A set of arguments for the binding. The syntax and semantics of these arguments - depends on the exchange class. - - - - - - This method confirms that the bind was successful. - - - - - - - - This method unbinds a queue from an exchange. - - If a unbind fails, the server MUST raise a connection exception. - - - - - - - - - Specifies the name of the queue to unbind. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to unbind an unnamed queue. - - - - - The client MUST NOT attempt to unbind a queue that does not exist. - - - The client attempts to unbind a non-existent queue. - - - - - - The name of the exchange to unbind from. - - - The client MUST NOT attempt to unbind a queue from an exchange that - does not exist. - - - The client attempts to unbind a queue from a non-existent exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - - Specifies the routing key of the binding to unbind. - - - - Specifies the arguments of the binding to unbind. - - - - - This method confirms that the unbind was successful. - - - - - - - - This method removes all messages from a queue which are not awaiting - acknowledgment. - - - - - The server MUST NOT purge messages that have already been sent to a client - but not yet acknowledged. - - - - - - The server MAY implement a purge queue or log that allows system administrators - to recover accidentally-purged messages. The server SHOULD NOT keep purged - messages in the same storage spaces as the live messages since the volumes of - purged messages may get very large. - - - - - - - - - - - - Specifies the name of the queue to purge. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to purge an unnamed queue. - - - - - The client MUST NOT attempt to purge a queue that does not exist. - - - The client attempts to purge a non-existent queue. - - - - - - - - - This method confirms the purge of a queue. - - - - - - Reports the number of messages purged. - - - - - - - - - This method deletes a queue. When a queue is deleted any pending messages are sent - to a dead-letter queue if this is defined in the server configuration, and all - consumers on the queue are cancelled. - - - - - The server SHOULD use a dead-letter queue to hold messages that were pending on - a deleted queue, and MAY provide facilities for a system administrator to move - these messages back to an active queue. - - - - - - - - - - - - Specifies the name of the queue to delete. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to delete an unnamed queue. - - - - - The client MUST NOT attempt to delete a queue that does not exist. - - - The client attempts to delete a non-existent queue. - - - - - - - If set, the server will only delete the queue if it has no consumers. If the - queue has consumers the server does does not delete it but raises a channel - exception instead. - - - - The server MUST NOT delete a queue that has consumers on it, if the if-unused - field is true. - - - The client declares a queue, and consumes from it, then tries to delete it - setting if-unused to true. - - - - - - - If set, the server will only delete the queue if it has no messages. - - - - The server MUST NOT delete a queue that has messages on it, if the - if-empty field is true. - - - The client declares a queue, binds it and publishes some messages into it, - then tries to delete it setting if-empty to true. - - - - - - - - - This method confirms the deletion of a queue. - - - - - Reports the number of messages deleted. - - - - - - - - - The Basic class provides methods that support an industry-standard messaging model. - - - - basic = C:QOS S:QOS-OK - / C:CONSUME S:CONSUME-OK - / C:CANCEL S:CANCEL-OK - / C:PUBLISH content - / S:RETURN content - / S:DELIVER content - / C:GET ( S:GET-OK content / S:GET-EMPTY ) - / C:ACK - / C:REJECT - / C:RECOVER-ASYNC - / C:RECOVER S:RECOVER-OK - - - - - - - - The server SHOULD respect the persistent property of basic messages and - SHOULD make a best-effort to hold persistent basic messages on a reliable - storage mechanism. - - - Send a persistent message to queue, stop server, restart server and then - verify whether message is still present. Assumes that queues are durable. - Persistence without durable queues makes no sense. - - - - - - The server MUST NOT discard a persistent basic message in case of a queue - overflow. - - - Declare a queue overflow situation with persistent messages and verify that - messages do not get lost (presumably the server will write them to disk). - - - - - - The server MAY use the Channel.Flow method to slow or stop a basic message - publisher when necessary. - - - Declare a queue overflow situation with non-persistent messages and verify - whether the server responds with Channel.Flow or not. Repeat with persistent - messages. - - - - - - The server MAY overflow non-persistent basic messages to persistent - storage. - - - - - - - The server MAY discard or dead-letter non-persistent basic messages on a - priority basis if the queue size exceeds some configured limit. - - - - - - - The server MUST implement at least 2 priority levels for basic messages, - where priorities 0-4 and 5-9 are treated as two distinct levels. - - - Send a number of priority 0 messages to a queue. Send one priority 9 - message. Consume messages from the queue and verify that the first message - received was priority 9. - - - - - - The server MAY implement up to 10 priority levels. - - - Send a number of messages with mixed priorities to a queue, so that all - priority values from 0 to 9 are exercised. A good scenario would be ten - messages in low-to-high priority. Consume from queue and verify how many - priority levels emerge. - - - - - - The server MUST deliver messages of the same priority in order irrespective of - their individual persistence. - - - Send a set of messages with the same priority but different persistence - settings to a queue. Consume and verify that messages arrive in same order - as originally published. - - - - - - The server MUST support un-acknowledged delivery of Basic content, i.e. - consumers with the no-ack field set to TRUE. - - - - - - The server MUST support explicitly acknowledged delivery of Basic content, - i.e. consumers with the no-ack field set to FALSE. - - - Declare a queue and a consumer using explicit acknowledgements. Publish a - set of messages to the queue. Consume the messages but acknowledge only - half of them. Disconnect and reconnect, and consume from the queue. - Verify that the remaining messages are received. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This method requests a specific quality of service. The QoS can be specified for the - current channel or for all channels on the connection. The particular properties and - semantics of a qos method always depend on the content class semantics. Though the - qos method could in principle apply to both peers, it is currently meaningful only - for the server. - - - - - - - - The client can request that messages be sent in advance so that when the client - finishes processing a message, the following message is already held locally, - rather than needing to be sent down the channel. Prefetching gives a performance - improvement. This field specifies the prefetch window size in octets. The server - will send a message in advance if it is equal to or smaller in size than the - available prefetch size (and also falls into other prefetch limits). May be set - to zero, meaning "no specific limit", although other prefetch limits may still - apply. The prefetch-size is ignored if the no-ack option is set. - - - - The server MUST ignore this setting when the client is not processing any - messages - i.e. the prefetch size does not limit the transfer of single - messages to a client, only the sending in advance of more messages while - the client still has one or more unacknowledged messages. - - - Define a QoS prefetch-size limit and send a single message that exceeds - that limit. Verify that the message arrives correctly. - - - - - - - Specifies a prefetch window in terms of whole messages. This field may be used - in combination with the prefetch-size field; a message will only be sent in - advance if both prefetch windows (and those at the channel and connection level) - allow it. The prefetch-count is ignored if the no-ack option is set. - - - - The server may send less data in advance than allowed by the client's - specified prefetch windows but it MUST NOT send more. - - - Define a QoS prefetch-size limit and a prefetch-count limit greater than - one. Send multiple messages that exceed the prefetch size. Verify that - no more than one message arrives at once. - - - - - - - By default the QoS settings apply to the current channel only. If this field is - set, they are applied to the entire connection. - - - - - - - This method tells the client that the requested QoS levels could be handled by the - server. The requested QoS applies to all active consumers until a new QoS is - defined. - - - - - - - - - This method asks the server to start a "consumer", which is a transient request for - messages from a specific queue. Consumers last as long as the channel they were - declared on, or until the client cancels them. - - - - - The server SHOULD support at least 16 consumers per queue, and ideally, impose - no limit except as defined by available resources. - - - Declare a queue and create consumers on that queue until the server closes the - connection. Verify that the number of consumers created was at least sixteen - and report the total number. - - - - - - - - - - - Specifies the name of the queue to consume from. - - - - - Specifies the identifier for the consumer. The consumer tag is local to a - channel, so two clients can use the same consumer tags. If this field is - empty the server will generate a unique tag. - - - - The client MUST NOT specify a tag that refers to an existing consumer. - - - Attempt to create two consumers with the same non-empty tag, on the - same channel. - - - - - The consumer tag is valid only within the channel from which the - consumer was created. I.e. a client MUST NOT create a consumer in one - channel and then use it in another. - - - Attempt to create a consumer in one channel, then use in another channel, - in which consumers have also been created (to test that the server uses - unique consumer tags). - - - - - - - - - - - Request exclusive consumer access, meaning only this consumer can access the - queue. - - - - - The client MAY NOT gain exclusive access to a queue that already has - active consumers. - - - Open two connections to a server, and in one connection declare a shared - (non-exclusive) queue and then consume from the queue. In the second - connection attempt to consume from the same queue using the exclusive - option. - - - - - - - - - A set of arguments for the consume. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - The server provides the client with a consumer tag, which is used by the client - for methods called on the consumer at a later stage. - - - - - Holds the consumer tag specified by the client or provided by the server. - - - - - - - - - This method cancels a consumer. This does not affect already delivered - messages, but it does mean the server will not send any more messages for - that consumer. The client may receive an arbitrary number of messages in - between sending the cancel method and receiving the cancel-ok reply. - - - - - If the queue does not exist the server MUST ignore the cancel method, so - long as the consumer tag is valid for that channel. - - - TODO. - - - - - - - - - - - - - This method confirms that the cancellation was completed. - - - - - - - - - - This method publishes a message to a specific exchange. The message will be routed - to queues as defined by the exchange configuration and distributed to any active - consumers when the transaction, if any, is committed. - - - - - - - - - - Specifies the name of the exchange to publish to. The exchange name can be - empty, meaning the default exchange. If the exchange name is specified, and that - exchange does not exist, the server will raise a channel exception. - - - - - The client MUST NOT attempt to publish a content to an exchange that - does not exist. - - - The client attempts to publish a content to a non-existent exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - If the exchange was declared as an internal exchange, the server MUST raise - a channel exception with a reply code 403 (access refused). - - - TODO. - - - - - - The exchange MAY refuse basic content in which case it MUST raise a channel - exception with reply code 540 (not implemented). - - - TODO. - - - - - - - Specifies the routing key for the message. The routing key is used for routing - messages depending on the exchange configuration. - - - - - - This flag tells the server how to react if the message cannot be routed to a - queue. If this flag is set, the server will return an unroutable message with a - Return method. If this flag is zero, the server silently drops the message. - - - - - The server SHOULD implement the mandatory flag. - - - TODO. - - - - - - - This flag tells the server how to react if the message cannot be routed to a - queue consumer immediately. If this flag is set, the server will return an - undeliverable message with a Return method. If this flag is zero, the server - will queue the message, but with no guarantee that it will ever be consumed. - - - - - The server SHOULD implement the immediate flag. - - - TODO. - - - - - - - - This method returns an undeliverable message that was published with the "immediate" - flag set, or an unroutable message published with the "mandatory" flag set. The - reply code and text provide information about the reason that the message was - undeliverable. - - - - - - - - - - Specifies the name of the exchange that the message was originally published - to. May be empty, meaning the default exchange. - - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - - This method delivers a message to the client, via a consumer. In the asynchronous - message delivery model, the client starts a consumer using the Consume method, then - the server responds with Deliver methods as and when messages arrive for that - consumer. - - - - - The server SHOULD track the number of times a message has been delivered to - clients and when a message is redelivered a certain number of times - e.g. 5 - times - without being acknowledged, the server SHOULD consider the message to be - unprocessable (possibly causing client applications to abort), and move the - message to a dead letter queue. - - - TODO. - - - - - - - - - - - - Specifies the name of the exchange that the message was originally published to. - May be empty, indicating the default exchange. - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - This method provides a direct access to the messages in a queue using a synchronous - dialogue that is designed for specific types of application where synchronous - functionality is more important than performance. - - - - - - - - - - - Specifies the name of the queue to get a message from. - - - - - - - This method delivers a message to the client following a get method. A message - delivered by 'get-ok' must be acknowledged unless the no-ack option was set in the - get method. - - - - - - - - - Specifies the name of the exchange that the message was originally published to. - If empty, the message was published to the default exchange. - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - This method tells the client that the queue has no messages available for the - client. - - - - - - - - - - - This method acknowledges one or more messages delivered via the Deliver or Get-Ok - methods. The client can ask to confirm a single message or a set of messages up to - and including a specific message. - - - - - - - - If set to 1, the delivery tag is treated as "up to and including", so that the - client can acknowledge multiple messages with a single method. If set to zero, - the delivery tag refers to a single message. If the multiple field is 1, and the - delivery tag is zero, tells the server to acknowledge all outstanding messages. - - - - The server MUST validate that a non-zero delivery-tag refers to a delivered - message, and raise a channel exception if this is not the case. On a transacted - channel, this check MUST be done immediately and not delayed until a Tx.Commit. - Specifically, a client MUST not acknowledge the same message more than once. - - - TODO. - - - - - - - - - - This method allows a client to reject a message. It can be used to interrupt and - cancel large incoming messages, or return untreatable messages to their original - queue. - - - - - The server SHOULD be capable of accepting and process the Reject method while - sending message content with a Deliver or Get-Ok method. I.e. the server should - read and process incoming methods while sending output frames. To cancel a - partially-send content, the server sends a content body frame of size 1 (i.e. - with no data except the frame-end octet). - - - - - - The server SHOULD interpret this method as meaning that the client is unable to - process the message at this time. - - - TODO. - - - - - - The client MUST NOT use this method as a means of selecting messages to process. - - - TODO. - - - - - - - - - - If requeue is true, the server will attempt to requeue the message. If requeue - is false or the requeue attempt fails the messages are discarded or dead-lettered. - - - - - The server MUST NOT deliver the message to the same client within the - context of the current channel. The recommended strategy is to attempt to - deliver the message to an alternative consumer, and if that is not possible, - to move the message to a dead-letter queue. The server MAY use more - sophisticated tracking to hold the message on the queue and redeliver it to - the same client at a later stage. - - - TODO. - - - - - - - - - - This method asks the server to redeliver all unacknowledged messages on a - specified channel. Zero or more messages may be redelivered. This method - is deprecated in favour of the synchronous Recover/Recover-Ok. - - - - The server MUST set the redelivered flag on all messages that are resent. - - - TODO. - - - - - - If this field is zero, the message will be redelivered to the original - recipient. If this bit is 1, the server will attempt to requeue the message, - potentially then delivering it to an alternative subscriber. - - - - - - - - - This method asks the server to redeliver all unacknowledged messages on a - specified channel. Zero or more messages may be redelivered. This method - replaces the asynchronous Recover. - - - - The server MUST set the redelivered flag on all messages that are resent. - - - TODO. - - - - - - If this field is zero, the message will be redelivered to the original - recipient. If this bit is 1, the server will attempt to requeue the message, - potentially then delivering it to an alternative subscriber. - - - - - - - This method acknowledges a Basic.Recover method. - - - - - - - - - - The Tx class allows publish and ack operations to be batched into atomic - units of work. The intention is that all publish and ack requests issued - within a transaction will complete successfully or none of them will. - Servers SHOULD implement atomic transactions at least where all publish - or ack requests affect a single queue. Transactions that cover multiple - queues may be non-atomic, given that queues can be created and destroyed - asynchronously, and such events do not form part of any transaction. - Further, the behaviour of transactions with respect to the immediate and - mandatory flags on Basic.Publish methods is not defined. - - - - - Applications MUST NOT rely on the atomicity of transactions that - affect more than one queue. - - - - - Applications MUST NOT rely on the behaviour of transactions that - include messages published with the immediate option. - - - - - Applications MUST NOT rely on the behaviour of transactions that - include messages published with the mandatory option. - - - - - tx = C:SELECT S:SELECT-OK - / C:COMMIT S:COMMIT-OK - / C:ROLLBACK S:ROLLBACK-OK - - - - - - - - - - This method sets the channel to use standard transactions. The client must use this - method at least once on a channel before using the Commit or Rollback methods. - - - - - - - - This method confirms to the client that the channel was successfully set to use - standard transactions. - - - - - - - - - This method commits all message publications and acknowledgments performed in - the current transaction. A new transaction starts immediately after a commit. - - - - - - - The client MUST NOT use the Commit method on non-transacted channels. - - - The client opens a channel and then uses Tx.Commit. - - - - - - - This method confirms to the client that the commit succeeded. Note that if a commit - fails, the server raises a channel exception. - - - - - - - - - This method abandons all message publications and acknowledgments performed in - the current transaction. A new transaction starts immediately after a rollback. - Note that unacked messages will not be automatically redelivered by rollback; - if that is required an explicit recover call should be issued. - - - - - - - The client MUST NOT use the Rollback method on non-transacted channels. - - - The client opens a channel and then uses Tx.Rollback. - - - - - - - This method confirms to the client that the rollback succeeded. Note that if an - rollback fails, the server raises a channel exception. - - - - - - + + + + + + + + + + + + + + + + + + + + + + + Indicates that the method completed successfully. This reply code is + reserved for future use - the current protocol design does not use positive + confirmation and reply codes are sent only in case of an error. + + + + + + The client attempted to transfer content larger than the server could accept + at the present time. The client may retry at a later time. + + + + + + When the exchange cannot deliver to a consumer when the immediate flag is + set. As a result of pending data on the queue or the absence of any + consumers of the queue. + + + + + + An operator intervened to close the connection for some reason. The client + may retry at some later date. + + + + + + The client tried to work with an unknown virtual host. + + + + + + The client attempted to work with a server entity to which it has no + access due to security settings. + + + + + + The client attempted to work with a server entity that does not exist. + + + + + + The client attempted to work with a server entity to which it has no + access because another client is working with it. + + + + + + The client requested a method that was not allowed because some precondition + failed. + + + + + + The sender sent a malformed frame that the recipient could not decode. + This strongly implies a programming error in the sending peer. + + + + + + The sender sent a frame that contained illegal values for one or more + fields. This strongly implies a programming error in the sending peer. + + + + + + The client sent an invalid sequence of frames, attempting to perform an + operation that was considered invalid by the server. This usually implies + a programming error in the client. + + + + + + The client attempted to work with a channel that had not been correctly + opened. This most likely indicates a fault in the client layer. + + + + + + The peer sent a frame that was not expected, usually in the context of + a content header and body. This strongly indicates a fault in the peer's + content processing. + + + + + + The server could not complete the method because it lacked sufficient + resources. This may be due to the client creating too many of some type + of entity. + + + + + + The client tried to work with some entity in a manner that is prohibited + by the server, due to security settings or by some other criteria. + + + + + + The client tried to use functionality that is not implemented in the + server. + + + + + + The server could not complete the method because of an internal error. + The server may require intervention by an operator in order to resume + normal operations. + + + + + + + + + + Identifier for the consumer, valid within the current channel. + + + + + + The server-assigned and channel-specific delivery tag + + + + The delivery tag is valid only within the channel from which the message was + received. I.e. a client MUST NOT receive a message on one channel and then + acknowledge it on another. + + + + + The server MUST NOT use a zero value for delivery tags. Zero is reserved + for client use, meaning "all messages so far received". + + + + + + + The exchange name is a client-selected string that identifies the exchange for + publish methods. + + + + + + + + + + If this field is set the server does not expect acknowledgements for + messages. That is, when a message is delivered to the client the server + assumes the delivery will succeed and immediately dequeues it. This + functionality may increase performance but at the cost of reliability. + Messages can get lost if a client dies before they are delivered to the + application. + + + + + + If the no-local field is set the server will not send messages to the connection that + published them. + + + + + + If set, the server will not respond to the method. The client should not wait + for a reply method. If the server could not complete the method it will raise a + channel or connection exception. + + + + + + Unconstrained. + + + + + + + + This table provides a set of peer properties, used for identification, debugging, + and general information. + + + + + + The queue name identifies the queue within the vhost. In methods where the queue + name may be blank, and that has no specific significance, this refers to the + 'current' queue for the channel, meaning the last queue that the client declared + on the channel. If the client did not declare a queue, and the method needs a + queue name, this will result in a 502 (syntax error) channel exception. + + + + + + + + This indicates that the message has been previously delivered to this or + another client. + + + + The server SHOULD try to signal redelivered messages when it can. When + redelivering a message that was not successfully acknowledged, the server + SHOULD deliver it to the original client if possible. + + + Declare a shared queue and publish a message to the queue. Consume the + message using explicit acknowledgements, but do not acknowledge the + message. Close the connection, reconnect, and consume from the queue + again. The message should arrive with the redelivered flag set. + + + + + The client MUST NOT rely on the redelivered field but should take it as a + hint that the message may already have been processed. A fully robust + client must be able to track duplicate received messages on non-transacted, + and locally-transacted channels. + + + + + + + The number of messages in the queue, which will be zero for newly-declared + queues. This is the number of messages present in the queue, and committed + if the channel on which they were published is transacted, that are not + waiting acknowledgement. + + + + + + The reply code. The AMQ reply codes are defined as constants at the start + of this formal specification. + + + + + + + The localised reply text. This text can be logged as an aid to resolving + issues. + + + + + + + + + + + + + + + + + + + + The connection class provides methods for a client to establish a network connection to + a server, and for both peers to operate the connection thereafter. + + + + connection = open-connection *use-connection close-connection + open-connection = C:protocol-header + S:START C:START-OK + *challenge + S:TUNE C:TUNE-OK + C:OPEN S:OPEN-OK + challenge = S:SECURE C:SECURE-OK + use-connection = *channel + close-connection = C:CLOSE S:CLOSE-OK + / S:CLOSE C:CLOSE-OK + + + + + + + + + + This method starts the connection negotiation process by telling the client the + protocol version that the server proposes, along with a list of security mechanisms + which the client can use for authentication. + + + + + If the server cannot support the protocol specified in the protocol header, + it MUST respond with a valid protocol header and then close the socket + connection. + + + The client sends a protocol header containing an invalid protocol name. + The server MUST respond by sending a valid protocol header and then closing + the connection. + + + + + The server MUST provide a protocol version that is lower than or equal to + that requested by the client in the protocol header. + + + The client requests a protocol version that is higher than any valid + implementation, e.g. 2.0. The server must respond with a protocol header + indicating its supported protocol version, e.g. 1.0. + + + + + If the client cannot handle the protocol version suggested by the server + it MUST close the socket connection without sending any further data. + + + The server sends a protocol version that is lower than any valid + implementation, e.g. 0.1. The client must respond by closing the + connection without sending any further data. + + + + + + + + + The major version number can take any value from 0 to 99 as defined in the + AMQP specification. + + + + + + The minor version number can take any value from 0 to 99 as defined in the + AMQP specification. + + + + + + + The properties SHOULD contain at least these fields: "host", specifying the + server host name or address, "product", giving the name of the server product, + "version", giving the name of the server version, "platform", giving the name + of the operating system, "copyright", if appropriate, and "information", giving + other general information. + + + Client connects to server and inspects the server properties. It checks for + the presence of the required fields. + + + + + + + A list of the security mechanisms that the server supports, delimited by spaces. + + + + + + + A list of the message locales that the server supports, delimited by spaces. The + locale defines the language in which the server will send reply texts. + + + + The server MUST support at least the en_US locale. + + + Client connects to server and inspects the locales field. It checks for + the presence of the required locale(s). + + + + + + + + + This method selects a SASL security mechanism. + + + + + + + + + The properties SHOULD contain at least these fields: "product", giving the name + of the client product, "version", giving the name of the client version, "platform", + giving the name of the operating system, "copyright", if appropriate, and + "information", giving other general information. + + + + + + + A single security mechanisms selected by the client, which must be one of those + specified by the server. + + + + The client SHOULD authenticate using the highest-level security profile it + can handle from the list provided by the server. + + + + + If the mechanism field does not contain one of the security mechanisms + proposed by the server in the Start method, the server MUST close the + connection without sending any further data. + + + Client connects to server and sends an invalid security mechanism. The + server must respond by closing the connection (a socket close, with no + connection close negotiation). + + + + + + + + A block of opaque data passed to the security mechanism. The contents of this + data are defined by the SASL security mechanism. + + + + + + + A single message locale selected by the client, which must be one of those + specified by the server. + + + + + + + + + + The SASL protocol works by exchanging challenges and responses until both peers have + received sufficient information to authenticate each other. This method challenges + the client to provide more information. + + + + + + + + Challenge information, a block of opaque binary data passed to the security + mechanism. + + + + + + + This method attempts to authenticate, passing a block of SASL data for the security + mechanism at the server side. + + + + + + + A block of opaque data passed to the security mechanism. The contents of this + data are defined by the SASL security mechanism. + + + + + + + + + + This method proposes a set of connection configuration values to the client. The + client can accept and/or adjust these. + + + + + + + + + Specifies highest channel number that the server permits. Usable channel numbers + are in the range 1..channel-max. Zero indicates no specified limit. + + + + + + The largest frame size that the server proposes for the connection, including + frame header and end-byte. The client can negotiate a lower value. Zero means + that the server does not impose any specific limit but may reject very large + frames if it cannot allocate resources for them. + + + + Until the frame-max has been negotiated, both peers MUST accept frames of up + to frame-min-size octets large, and the minimum negotiated value for frame-max + is also frame-min-size. + + + Client connects to server and sends a large properties field, creating a frame + of frame-min-size octets. The server must accept this frame. + + + + + + + The delay, in seconds, of the connection heartbeat that the server wants. + Zero means the server does not want a heartbeat. + + + + + + + This method sends the client's connection tuning parameters to the server. + Certain fields are negotiated, others provide capability information. + + + + + + + The maximum total number of channels that the client will use per connection. + + + + If the client specifies a channel max that is higher than the value provided + by the server, the server MUST close the connection without attempting a + negotiated close. The server may report the error in some fashion to assist + implementors. + + + + + + + + + The largest frame size that the client and server will use for the connection. + Zero means that the client does not impose any specific limit but may reject + very large frames if it cannot allocate resources for them. Note that the + frame-max limit applies principally to content frames, where large contents can + be broken into frames of arbitrary size. + + + + Until the frame-max has been negotiated, both peers MUST accept frames of up + to frame-min-size octets large, and the minimum negotiated value for frame-max + is also frame-min-size. + + + + + If the client specifies a frame max that is higher than the value provided + by the server, the server MUST close the connection without attempting a + negotiated close. The server may report the error in some fashion to assist + implementors. + + + + + + + The delay, in seconds, of the connection heartbeat that the client wants. Zero + means the client does not want a heartbeat. + + + + + + + + + This method opens a connection to a virtual host, which is a collection of + resources, and acts to separate multiple application domains within a server. + The server may apply arbitrary limits per virtual host, such as the number + of each type of entity that may be used, per connection and/or in total. + + + + + + + + The name of the virtual host to work with. + + + + If the server supports multiple virtual hosts, it MUST enforce a full + separation of exchanges, queues, and all associated entities per virtual + host. An application, connected to a specific virtual host, MUST NOT be able + to access resources of another virtual host. + + + + + The server SHOULD verify that the client has permission to access the + specified virtual host. + + + + + + + + + + + + This method signals to the client that the connection is ready for use. + + + + + + + + + + + This method indicates that the sender wants to close the connection. This may be + due to internal conditions (e.g. a forced shut-down) or due to an error handling + a specific method, i.e. an exception. When a close is due to an exception, the + sender provides the class and method id of the method which caused the exception. + + + + After sending this method, any received methods except Close and Close-OK MUST + be discarded. The response to receiving a Close after sending Close must be to + send Close-Ok. + + + + + + + + + + + + + When the close is provoked by a method exception, this is the class of the + method. + + + + + + When the close is provoked by a method exception, this is the ID of the method. + + + + + + + This method confirms a Connection.Close method and tells the recipient that it is + safe to release resources for the connection and close the socket. + + + + A peer that detects a socket closure without having received a Close-Ok + handshake method SHOULD log the error. + + + + + + + + + + + + The channel class provides methods for a client to establish a channel to a + server and for both peers to operate the channel thereafter. + + + + channel = open-channel *use-channel close-channel + open-channel = C:OPEN S:OPEN-OK + use-channel = C:FLOW S:FLOW-OK + / S:FLOW C:FLOW-OK + / functional-class + close-channel = C:CLOSE S:CLOSE-OK + / S:CLOSE C:CLOSE-OK + + + + + + + + + + This method opens a channel to the server. + + + + The client MUST NOT use this method on an already-opened channel. + + + Client opens a channel and then reopens the same channel. + + + + + + + + + + + This method signals to the client that the channel is ready for use. + + + + + + + + + + + This method asks the peer to pause or restart the flow of content data sent by + a consumer. This is a simple flow-control mechanism that a peer can use to avoid + overflowing its queues or otherwise finding itself receiving more messages than + it can process. Note that this method is not intended for window control. It does + not affect contents returned by Basic.Get-Ok methods. + + + + + When a new channel is opened, it is active (flow is active). Some applications + assume that channels are inactive until started. To emulate this behaviour a + client MAY open the channel, then pause it. + + + + + + When sending content frames, a peer SHOULD monitor the channel for incoming + methods and respond to a Channel.Flow as rapidly as possible. + + + + + + A peer MAY use the Channel.Flow method to throttle incoming content data for + internal reasons, for example, when exchanging data over a slower connection. + + + + + + The peer that requests a Channel.Flow method MAY disconnect and/or ban a peer + that does not respect the request. This is to prevent badly-behaved clients + from overwhelming a server. + + + + + + + + + + + If 1, the peer starts sending content frames. If 0, the peer stops sending + content frames. + + + + + + + Confirms to the peer that a flow command was received and processed. + + + + + + Confirms the setting of the processed flow method: 1 means the peer will start + sending or continue to send content frames; 0 means it will not. + + + + + + + + + This method indicates that the sender wants to close the channel. This may be due to + internal conditions (e.g. a forced shut-down) or due to an error handling a specific + method, i.e. an exception. When a close is due to an exception, the sender provides + the class and method id of the method which caused the exception. + + + + After sending this method, any received methods except Close and Close-OK MUST + be discarded. The response to receiving a Close after sending Close must be to + send Close-Ok. + + + + + + + + + + + + + When the close is provoked by a method exception, this is the class of the + method. + + + + + + When the close is provoked by a method exception, this is the ID of the method. + + + + + + + This method confirms a Channel.Close method and tells the recipient that it is safe + to release resources for the channel. + + + + A peer that detects a socket closure without having received a Channel.Close-Ok + handshake method SHOULD log the error. + + + + + + + + + + + + Exchanges match and distribute messages across queues. Exchanges can be configured in + the server or declared at runtime. + + + + exchange = C:DECLARE S:DECLARE-OK + / C:DELETE S:DELETE-OK + + + + + + + + The server MUST implement these standard exchange types: fanout, direct. + + + Client attempts to declare an exchange with each of these standard types. + + + + + The server SHOULD implement these standard exchange types: topic, headers. + + + Client attempts to declare an exchange with each of these standard types. + + + + + The server MUST, in each virtual host, pre-declare an exchange instance + for each standard exchange type that it implements, where the name of the + exchange instance, if defined, is "amq." followed by the exchange type name. + + + The server MUST, in each virtual host, pre-declare at least two direct + exchange instances: one named "amq.direct", the other with no public name + that serves as a default exchange for Publish methods. + + + Client declares a temporary queue and attempts to bind to each required + exchange instance ("amq.fanout", "amq.direct", "amq.topic", and "amq.headers" + if those types are defined). + + + + + The server MUST pre-declare a direct exchange with no public name to act as + the default exchange for content Publish methods and for default queue bindings. + + + Client checks that the default exchange is active by specifying a queue + binding with no exchange name, and publishing a message with a suitable + routing key but without specifying the exchange name, then ensuring that + the message arrives in the queue correctly. + + + + + The server MUST NOT allow clients to access the default exchange except + by specifying an empty exchange name in the Queue.Bind and content Publish + methods. + + + + + The server MAY implement other exchange types as wanted. + + + + + + + + This method creates an exchange if it does not already exist, and if the exchange + exists, verifies that it is of the correct and expected class. + + + + The server SHOULD support a minimum of 16 exchanges per virtual host and + ideally, impose no limit except as defined by available resources. + + + The client declares as many exchanges as it can until the server reports + an error; the number of exchanges successfully declared must be at least + sixteen. + + + + + + + + + + + + + Exchange names starting with "amq." are reserved for pre-declared and + standardised exchanges. The client MAY declare an exchange starting with + "amq." if the passive option is set, or the exchange already exists. + + + The client attempts to declare a non-existing exchange starting with + "amq." and with the passive option set to zero. + + + + + The exchange name consists of a non-empty sequence of these characters: + letters, digits, hyphen, underscore, period, or colon. + + + The client attempts to declare an exchange with an illegal name. + + + + + + + + Each exchange belongs to one of a set of exchange types implemented by the + server. The exchange types define the functionality of the exchange - i.e. how + messages are routed through it. It is not valid or meaningful to attempt to + change the type of an existing exchange. + + + + Exchanges cannot be redeclared with different types. The client MUST not + attempt to redeclare an existing exchange with a different type than used + in the original Exchange.Declare method. + + + TODO. + + + + + The client MUST NOT attempt to declare an exchange with a type that the + server does not support. + + + TODO. + + + + + + + If set, the server will reply with Declare-Ok if the exchange already + exists with the same name, and raise an error if not. The client can + use this to check whether an exchange exists without modifying the + server state. When set, all other method fields except name and no-wait + are ignored. A declare with both passive and no-wait has no effect. + Arguments are compared for semantic equivalence. + + + + If set, and the exchange does not already exist, the server MUST + raise a channel exception with reply code 404 (not found). + + + TODO. + + + + + If not set and the exchange exists, the server MUST check that the + existing exchange has the same values for type, durable, and arguments + fields. The server MUST respond with Declare-Ok if the requested + exchange matches these fields, and MUST raise a channel exception if + not. + + + TODO. + + + + + + + If set when creating a new exchange, the exchange will be marked as durable. + Durable exchanges remain active when a server restarts. Non-durable exchanges + (transient exchanges) are purged if/when a server restarts. + + + + The server MUST support both durable and transient exchanges. + + + TODO. + + + + + + + + + + + + + A set of arguments for the declaration. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + This method confirms a Declare method and confirms the name of the exchange, + essential for automatically-named exchanges. + + + + + + + + + This method deletes an exchange. When an exchange is deleted all queue bindings on + the exchange are cancelled. + + + + + + + + + + + + The client MUST NOT attempt to delete an exchange that does not exist. + + + + + + + + If set, the server will only delete the exchange if it has no queue bindings. If + the exchange has queue bindings the server does not delete it but raises a + channel exception instead. + + + + The server MUST NOT delete an exchange that has bindings on it, if the if-unused + field is true. + + + The client declares an exchange, binds a queue to it, then tries to delete it + setting if-unused to true. + + + + + + + + + This method confirms the deletion of an exchange. + + + + + + + + + Queues store and forward messages. Queues can be configured in the server or created at + runtime. Queues must be attached to at least one exchange in order to receive messages + from publishers. + + + + queue = C:DECLARE S:DECLARE-OK + / C:BIND S:BIND-OK + / C:UNBIND S:UNBIND-OK + / C:PURGE S:PURGE-OK + / C:DELETE S:DELETE-OK + + + + + + + + + + This method creates or checks a queue. When creating a new queue the client can + specify various properties that control the durability of the queue and its + contents, and the level of sharing for the queue. + + + + + The server MUST create a default binding for a newly-declared queue to the + default exchange, which is an exchange of type 'direct' and use the queue + name as the routing key. + + + Client declares a new queue, and then without explicitly binding it to an + exchange, attempts to send a message through the default exchange binding, + i.e. publish a message to the empty exchange, with the queue name as routing + key. + + + + + + The server SHOULD support a minimum of 256 queues per virtual host and ideally, + impose no limit except as defined by available resources. + + + Client attempts to declare as many queues as it can until the server reports + an error. The resulting count must at least be 256. + + + + + + + + + + + + + The queue name MAY be empty, in which case the server MUST create a new + queue with a unique generated name and return this to the client in the + Declare-Ok method. + + + Client attempts to declare several queues with an empty name. The client then + verifies that the server-assigned names are unique and different. + + + + + Queue names starting with "amq." are reserved for pre-declared and + standardised queues. The client MAY declare a queue starting with + "amq." if the passive option is set, or the queue already exists. + + + The client attempts to declare a non-existing queue starting with + "amq." and with the passive option set to zero. + + + + + The queue name can be empty, or a sequence of these characters: + letters, digits, hyphen, underscore, period, or colon. + + + The client attempts to declare a queue with an illegal name. + + + + + + + If set, the server will reply with Declare-Ok if the queue already + exists with the same name, and raise an error if not. The client can + use this to check whether a queue exists without modifying the + server state. When set, all other method fields except name and no-wait + are ignored. A declare with both passive and no-wait has no effect. + Arguments are compared for semantic equivalence. + + + + The client MAY ask the server to assert that a queue exists without + creating the queue if not. If the queue does not exist, the server + treats this as a failure. + + + Client declares an existing queue with the passive option and expects + the server to respond with a declare-ok. Client then attempts to declare + a non-existent queue with the passive option, and the server must close + the channel with the correct reply-code. + + + + + If not set and the queue exists, the server MUST check that the + existing queue has the same values for durable, exclusive, auto-delete, + and arguments fields. The server MUST respond with Declare-Ok if the + requested queue matches these fields, and MUST raise a channel exception + if not. + + + TODO. + + + + + + + If set when creating a new queue, the queue will be marked as durable. Durable + queues remain active when a server restarts. Non-durable queues (transient + queues) are purged if/when a server restarts. Note that durable queues do not + necessarily hold persistent messages, although it does not make sense to send + persistent messages to a transient queue. + + + + The server MUST recreate the durable queue after a restart. + + + Client declares a durable queue. The server is then restarted. The client + then attempts to send a message to the queue. The message should be successfully + delivered. + + + + + The server MUST support both durable and transient queues. + + A client declares two named queues, one durable and one transient. + + + + + + + Exclusive queues may only be accessed by the current connection, and are + deleted when that connection closes. Passive declaration of an exclusive + queue by other connections are not allowed. + + + + + The server MUST support both exclusive (private) and non-exclusive (shared) + queues. + + + A client declares two named queues, one exclusive and one non-exclusive. + + + + + + The client MAY NOT attempt to use a queue that was declared as exclusive + by another still-open connection. + + + One client declares an exclusive queue. A second client on a different + connection attempts to declare, bind, consume, purge, delete, or declare + a queue of the same name. + + + + + + + If set, the queue is deleted when all consumers have finished using it. The last + consumer can be cancelled either explicitly or because its channel is closed. If + there was no consumer ever on the queue, it won't be deleted. Applications can + explicitly delete auto-delete queues using the Delete method as normal. + + + + + The server MUST ignore the auto-delete field if the queue already exists. + + + Client declares two named queues, one as auto-delete and one explicit-delete. + Client then attempts to declare the two queues using the same names again, + but reversing the value of the auto-delete field in each case. Verify that the + queues still exist with the original auto-delete flag values. + + + + + + + + + A set of arguments for the declaration. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + This method confirms a Declare method and confirms the name of the queue, essential + for automatically-named queues. + + + + + + + Reports the name of the queue. If the server generated a queue name, this field + contains that name. + + + + + + + + + Reports the number of active consumers for the queue. Note that consumers can + suspend activity (Channel.Flow) in which case they do not appear in this count. + + + + + + + + + This method binds a queue to an exchange. Until a queue is bound it will not + receive any messages. In a classic messaging model, store-and-forward queues + are bound to a direct exchange and subscription queues are bound to a topic + exchange. + + + + + A server MUST allow ignore duplicate bindings - that is, two or more bind + methods for a specific queue, with identical arguments - without treating these + as an error. + + + A client binds a named queue to an exchange. The client then repeats the bind + (with identical arguments). + + + + + + A server MUST not deliver the same message more than once to a queue, even if + the queue has multiple bindings that match the message. + + + A client declares a named queue and binds it using multiple bindings to the + amq.topic exchange. The client then publishes a message that matches all its + bindings. + + + + + + The server MUST allow a durable queue to bind to a transient exchange. + + + A client declares a transient exchange. The client then declares a named durable + queue and then attempts to bind the transient exchange to the durable queue. + + + + + + Bindings of durable queues to durable exchanges are automatically durable + and the server MUST restore such bindings after a server restart. + + + A server declares a named durable queue and binds it to a durable exchange. The + server is restarted. The client then attempts to use the queue/exchange combination. + + + + + + The server SHOULD support at least 4 bindings per queue, and ideally, impose no + limit except as defined by available resources. + + + A client declares a named queue and attempts to bind it to 4 different + exchanges. + + + + + + + + + + + + Specifies the name of the queue to bind. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to bind an unnamed queue. + + + + + The client MUST NOT attempt to bind a queue that does not exist. + + + The client attempts to bind a non-existent queue. + + + + + + + + A client MUST NOT be allowed to bind a queue to a non-existent exchange. + + + A client attempts to bind an named queue to a undeclared exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + + + Specifies the routing key for the binding. The routing key is used for routing + messages depending on the exchange configuration. Not all exchanges use a + routing key - refer to the specific exchange documentation. If the queue name + is empty, the server uses the last queue declared on the channel. If the + routing key is also empty, the server uses this queue name for the routing + key as well. If the queue name is provided but the routing key is empty, the + server does the binding with that empty routing key. The meaning of empty + routing keys depends on the exchange implementation. + + + + If a message queue binds to a direct exchange using routing key K and a + publisher sends the exchange a message with routing key R, then the message + MUST be passed to the message queue if K = R. + + + + + + + + + A set of arguments for the binding. The syntax and semantics of these arguments + depends on the exchange class. + + + + + + This method confirms that the bind was successful. + + + + + + + + This method unbinds a queue from an exchange. + + If a unbind fails, the server MUST raise a connection exception. + + + + + + + + + Specifies the name of the queue to unbind. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to unbind an unnamed queue. + + + + + The client MUST NOT attempt to unbind a queue that does not exist. + + + The client attempts to unbind a non-existent queue. + + + + + + The name of the exchange to unbind from. + + + The client MUST NOT attempt to unbind a queue from an exchange that + does not exist. + + + The client attempts to unbind a queue from a non-existent exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + + Specifies the routing key of the binding to unbind. + + + + Specifies the arguments of the binding to unbind. + + + + + This method confirms that the unbind was successful. + + + + + + + + This method removes all messages from a queue which are not awaiting + acknowledgment. + + + + + The server MUST NOT purge messages that have already been sent to a client + but not yet acknowledged. + + + + + + The server MAY implement a purge queue or log that allows system administrators + to recover accidentally-purged messages. The server SHOULD NOT keep purged + messages in the same storage spaces as the live messages since the volumes of + purged messages may get very large. + + + + + + + + + + + + Specifies the name of the queue to purge. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to purge an unnamed queue. + + + + + The client MUST NOT attempt to purge a queue that does not exist. + + + The client attempts to purge a non-existent queue. + + + + + + + + + This method confirms the purge of a queue. + + + + + + Reports the number of messages purged. + + + + + + + + + This method deletes a queue. When a queue is deleted any pending messages are sent + to a dead-letter queue if this is defined in the server configuration, and all + consumers on the queue are cancelled. + + + + + The server SHOULD use a dead-letter queue to hold messages that were pending on + a deleted queue, and MAY provide facilities for a system administrator to move + these messages back to an active queue. + + + + + + + + + + + + Specifies the name of the queue to delete. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to delete an unnamed queue. + + + + + The client MUST NOT attempt to delete a queue that does not exist. + + + The client attempts to delete a non-existent queue. + + + + + + + If set, the server will only delete the queue if it has no consumers. If the + queue has consumers the server does does not delete it but raises a channel + exception instead. + + + + The server MUST NOT delete a queue that has consumers on it, if the if-unused + field is true. + + + The client declares a queue, and consumes from it, then tries to delete it + setting if-unused to true. + + + + + + + If set, the server will only delete the queue if it has no messages. + + + + The server MUST NOT delete a queue that has messages on it, if the + if-empty field is true. + + + The client declares a queue, binds it and publishes some messages into it, + then tries to delete it setting if-empty to true. + + + + + + + + + This method confirms the deletion of a queue. + + + + + Reports the number of messages deleted. + + + + + + + + + The Basic class provides methods that support an industry-standard messaging model. + + + + basic = C:QOS S:QOS-OK + / C:CONSUME S:CONSUME-OK + / C:CANCEL S:CANCEL-OK + / C:PUBLISH content + / S:RETURN content + / S:DELIVER content + / C:GET ( S:GET-OK content / S:GET-EMPTY ) + / C:ACK + / C:REJECT + / C:RECOVER-ASYNC + / C:RECOVER S:RECOVER-OK + + + + + + + + The server SHOULD respect the persistent property of basic messages and + SHOULD make a best-effort to hold persistent basic messages on a reliable + storage mechanism. + + + Send a persistent message to queue, stop server, restart server and then + verify whether message is still present. Assumes that queues are durable. + Persistence without durable queues makes no sense. + + + + + + The server MUST NOT discard a persistent basic message in case of a queue + overflow. + + + Declare a queue overflow situation with persistent messages and verify that + messages do not get lost (presumably the server will write them to disk). + + + + + + The server MAY use the Channel.Flow method to slow or stop a basic message + publisher when necessary. + + + Declare a queue overflow situation with non-persistent messages and verify + whether the server responds with Channel.Flow or not. Repeat with persistent + messages. + + + + + + The server MAY overflow non-persistent basic messages to persistent + storage. + + + + + + + The server MAY discard or dead-letter non-persistent basic messages on a + priority basis if the queue size exceeds some configured limit. + + + + + + + The server MUST implement at least 2 priority levels for basic messages, + where priorities 0-4 and 5-9 are treated as two distinct levels. + + + Send a number of priority 0 messages to a queue. Send one priority 9 + message. Consume messages from the queue and verify that the first message + received was priority 9. + + + + + + The server MAY implement up to 10 priority levels. + + + Send a number of messages with mixed priorities to a queue, so that all + priority values from 0 to 9 are exercised. A good scenario would be ten + messages in low-to-high priority. Consume from queue and verify how many + priority levels emerge. + + + + + + The server MUST deliver messages of the same priority in order irrespective of + their individual persistence. + + + Send a set of messages with the same priority but different persistence + settings to a queue. Consume and verify that messages arrive in same order + as originally published. + + + + + + The server MUST support un-acknowledged delivery of Basic content, i.e. + consumers with the no-ack field set to TRUE. + + + + + + The server MUST support explicitly acknowledged delivery of Basic content, + i.e. consumers with the no-ack field set to FALSE. + + + Declare a queue and a consumer using explicit acknowledgements. Publish a + set of messages to the queue. Consume the messages but acknowledge only + half of them. Disconnect and reconnect, and consume from the queue. + Verify that the remaining messages are received. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method requests a specific quality of service. The QoS can be specified for the + current channel or for all channels on the connection. The particular properties and + semantics of a qos method always depend on the content class semantics. Though the + qos method could in principle apply to both peers, it is currently meaningful only + for the server. + + + + + + + + The client can request that messages be sent in advance so that when the client + finishes processing a message, the following message is already held locally, + rather than needing to be sent down the channel. Prefetching gives a performance + improvement. This field specifies the prefetch window size in octets. The server + will send a message in advance if it is equal to or smaller in size than the + available prefetch size (and also falls into other prefetch limits). May be set + to zero, meaning "no specific limit", although other prefetch limits may still + apply. The prefetch-size is ignored if the no-ack option is set. + + + + The server MUST ignore this setting when the client is not processing any + messages - i.e. the prefetch size does not limit the transfer of single + messages to a client, only the sending in advance of more messages while + the client still has one or more unacknowledged messages. + + + Define a QoS prefetch-size limit and send a single message that exceeds + that limit. Verify that the message arrives correctly. + + + + + + + Specifies a prefetch window in terms of whole messages. This field may be used + in combination with the prefetch-size field; a message will only be sent in + advance if both prefetch windows (and those at the channel and connection level) + allow it. The prefetch-count is ignored if the no-ack option is set. + + + + The server may send less data in advance than allowed by the client's + specified prefetch windows but it MUST NOT send more. + + + Define a QoS prefetch-size limit and a prefetch-count limit greater than + one. Send multiple messages that exceed the prefetch size. Verify that + no more than one message arrives at once. + + + + + + + By default the QoS settings apply to the current channel only. If this field is + set, they are applied to the entire connection. + + + + + + + This method tells the client that the requested QoS levels could be handled by the + server. The requested QoS applies to all active consumers until a new QoS is + defined. + + + + + + + + + This method asks the server to start a "consumer", which is a transient request for + messages from a specific queue. Consumers last as long as the channel they were + declared on, or until the client cancels them. + + + + + The server SHOULD support at least 16 consumers per queue, and ideally, impose + no limit except as defined by available resources. + + + Declare a queue and create consumers on that queue until the server closes the + connection. Verify that the number of consumers created was at least sixteen + and report the total number. + + + + + + + + + + + Specifies the name of the queue to consume from. + + + + + Specifies the identifier for the consumer. The consumer tag is local to a + channel, so two clients can use the same consumer tags. If this field is + empty the server will generate a unique tag. + + + + The client MUST NOT specify a tag that refers to an existing consumer. + + + Attempt to create two consumers with the same non-empty tag, on the + same channel. + + + + + The consumer tag is valid only within the channel from which the + consumer was created. I.e. a client MUST NOT create a consumer in one + channel and then use it in another. + + + Attempt to create a consumer in one channel, then use in another channel, + in which consumers have also been created (to test that the server uses + unique consumer tags). + + + + + + + + + + + Request exclusive consumer access, meaning only this consumer can access the + queue. + + + + + The client MAY NOT gain exclusive access to a queue that already has + active consumers. + + + Open two connections to a server, and in one connection declare a shared + (non-exclusive) queue and then consume from the queue. In the second + connection attempt to consume from the same queue using the exclusive + option. + + + + + + + + + A set of arguments for the consume. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + The server provides the client with a consumer tag, which is used by the client + for methods called on the consumer at a later stage. + + + + + Holds the consumer tag specified by the client or provided by the server. + + + + + + + + + This method cancels a consumer. This does not affect already delivered + messages, but it does mean the server will not send any more messages for + that consumer. The client may receive an arbitrary number of messages in + between sending the cancel method and receiving the cancel-ok reply. + + + + + If the queue does not exist the server MUST ignore the cancel method, so + long as the consumer tag is valid for that channel. + + + TODO. + + + + + + + + + + + + + This method confirms that the cancellation was completed. + + + + + + + + + + This method publishes a message to a specific exchange. The message will be routed + to queues as defined by the exchange configuration and distributed to any active + consumers when the transaction, if any, is committed. + + + + + + + + + + Specifies the name of the exchange to publish to. The exchange name can be + empty, meaning the default exchange. If the exchange name is specified, and that + exchange does not exist, the server will raise a channel exception. + + + + + The client MUST NOT attempt to publish a content to an exchange that + does not exist. + + + The client attempts to publish a content to a non-existent exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + If the exchange was declared as an internal exchange, the server MUST raise + a channel exception with a reply code 403 (access refused). + + + TODO. + + + + + + The exchange MAY refuse basic content in which case it MUST raise a channel + exception with reply code 540 (not implemented). + + + TODO. + + + + + + + Specifies the routing key for the message. The routing key is used for routing + messages depending on the exchange configuration. + + + + + + This flag tells the server how to react if the message cannot be routed to a + queue. If this flag is set, the server will return an unroutable message with a + Return method. If this flag is zero, the server silently drops the message. + + + + + The server SHOULD implement the mandatory flag. + + + TODO. + + + + + + + This flag tells the server how to react if the message cannot be routed to a + queue consumer immediately. If this flag is set, the server will return an + undeliverable message with a Return method. If this flag is zero, the server + will queue the message, but with no guarantee that it will ever be consumed. + + + + + The server SHOULD implement the immediate flag. + + + TODO. + + + + + + + + This method returns an undeliverable message that was published with the "immediate" + flag set, or an unroutable message published with the "mandatory" flag set. The + reply code and text provide information about the reason that the message was + undeliverable. + + + + + + + + + + Specifies the name of the exchange that the message was originally published + to. May be empty, meaning the default exchange. + + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + + This method delivers a message to the client, via a consumer. In the asynchronous + message delivery model, the client starts a consumer using the Consume method, then + the server responds with Deliver methods as and when messages arrive for that + consumer. + + + + + The server SHOULD track the number of times a message has been delivered to + clients and when a message is redelivered a certain number of times - e.g. 5 + times - without being acknowledged, the server SHOULD consider the message to be + unprocessable (possibly causing client applications to abort), and move the + message to a dead letter queue. + + + TODO. + + + + + + + + + + + + Specifies the name of the exchange that the message was originally published to. + May be empty, indicating the default exchange. + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + This method provides a direct access to the messages in a queue using a synchronous + dialogue that is designed for specific types of application where synchronous + functionality is more important than performance. + + + + + + + + + + + Specifies the name of the queue to get a message from. + + + + + + + This method delivers a message to the client following a get method. A message + delivered by 'get-ok' must be acknowledged unless the no-ack option was set in the + get method. + + + + + + + + + Specifies the name of the exchange that the message was originally published to. + If empty, the message was published to the default exchange. + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + This method tells the client that the queue has no messages available for the + client. + + + + + + + + + + + This method acknowledges one or more messages delivered via the Deliver or Get-Ok + methods. The client can ask to confirm a single message or a set of messages up to + and including a specific message. + + + + + + + + If set to 1, the delivery tag is treated as "up to and including", so that the + client can acknowledge multiple messages with a single method. If set to zero, + the delivery tag refers to a single message. If the multiple field is 1, and the + delivery tag is zero, tells the server to acknowledge all outstanding messages. + + + + The server MUST validate that a non-zero delivery-tag refers to a delivered + message, and raise a channel exception if this is not the case. On a transacted + channel, this check MUST be done immediately and not delayed until a Tx.Commit. + Specifically, a client MUST not acknowledge the same message more than once. + + + TODO. + + + + + + + + + + This method allows a client to reject a message. It can be used to interrupt and + cancel large incoming messages, or return untreatable messages to their original + queue. + + + + + The server SHOULD be capable of accepting and process the Reject method while + sending message content with a Deliver or Get-Ok method. I.e. the server should + read and process incoming methods while sending output frames. To cancel a + partially-send content, the server sends a content body frame of size 1 (i.e. + with no data except the frame-end octet). + + + + + + The server SHOULD interpret this method as meaning that the client is unable to + process the message at this time. + + + TODO. + + + + + + The client MUST NOT use this method as a means of selecting messages to process. + + + TODO. + + + + + + + + + + If requeue is true, the server will attempt to requeue the message. If requeue + is false or the requeue attempt fails the messages are discarded or dead-lettered. + + + + + The server MUST NOT deliver the message to the same client within the + context of the current channel. The recommended strategy is to attempt to + deliver the message to an alternative consumer, and if that is not possible, + to move the message to a dead-letter queue. The server MAY use more + sophisticated tracking to hold the message on the queue and redeliver it to + the same client at a later stage. + + + TODO. + + + + + + + + + + This method asks the server to redeliver all unacknowledged messages on a + specified channel. Zero or more messages may be redelivered. This method + is deprecated in favour of the synchronous Recover/Recover-Ok. + + + + The server MUST set the redelivered flag on all messages that are resent. + + + TODO. + + + + + + If this field is zero, the message will be redelivered to the original + recipient. If this bit is 1, the server will attempt to requeue the message, + potentially then delivering it to an alternative subscriber. + + + + + + + + + This method asks the server to redeliver all unacknowledged messages on a + specified channel. Zero or more messages may be redelivered. This method + replaces the asynchronous Recover. + + + + The server MUST set the redelivered flag on all messages that are resent. + + + TODO. + + + + + + If this field is zero, the message will be redelivered to the original + recipient. If this bit is 1, the server will attempt to requeue the message, + potentially then delivering it to an alternative subscriber. + + + + + + + This method acknowledges a Basic.Recover method. + + + + + + + + + + The Tx class allows publish and ack operations to be batched into atomic + units of work. The intention is that all publish and ack requests issued + within a transaction will complete successfully or none of them will. + Servers SHOULD implement atomic transactions at least where all publish + or ack requests affect a single queue. Transactions that cover multiple + queues may be non-atomic, given that queues can be created and destroyed + asynchronously, and such events do not form part of any transaction. + Further, the behaviour of transactions with respect to the immediate and + mandatory flags on Basic.Publish methods is not defined. + + + + + Applications MUST NOT rely on the atomicity of transactions that + affect more than one queue. + + + + + Applications MUST NOT rely on the behaviour of transactions that + include messages published with the immediate option. + + + + + Applications MUST NOT rely on the behaviour of transactions that + include messages published with the mandatory option. + + + + + tx = C:SELECT S:SELECT-OK + / C:COMMIT S:COMMIT-OK + / C:ROLLBACK S:ROLLBACK-OK + + + + + + + + + + This method sets the channel to use standard transactions. The client must use this + method at least once on a channel before using the Commit or Rollback methods. + + + + + + + + This method confirms to the client that the channel was successfully set to use + standard transactions. + + + + + + + + + This method commits all message publications and acknowledgments performed in + the current transaction. A new transaction starts immediately after a commit. + + + + + + + The client MUST NOT use the Commit method on non-transacted channels. + + + The client opens a channel and then uses Tx.Commit. + + + + + + + This method confirms to the client that the commit succeeded. Note that if a commit + fails, the server raises a channel exception. + + + + + + + + + This method abandons all message publications and acknowledgments performed in + the current transaction. A new transaction starts immediately after a rollback. + Note that unacked messages will not be automatically redelivered by rollback; + if that is required an explicit recover call should be issued. + + + + + + + The client MUST NOT use the Rollback method on non-transacted channels. + + + The client opens a channel and then uses Tx.Rollback. + + + + + + + This method confirms to the client that the rollback succeeded. Note that if an + rollback fails, the server raises a channel exception. + + + + + + diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py index 0cb0ff3..22dec0c 100644 --- a/src/asynqp/bases.py +++ b/src/asynqp/bases.py @@ -1,21 +1,21 @@ -class Sender(object): - def __init__(self, channel_id, protocol): - self.channel_id = channel_id - self.protocol = protocol - - def send_method(self, method): - self.protocol.send_method(self.channel_id, method) - - -class FrameHandler(object): - def __init__(self, synchroniser, sender): - self.synchroniser = synchroniser - self.sender = sender - - def handle(self, frame): - try: - meth = getattr(self, 'handle_' + type(frame).__name__) - except AttributeError: - meth = getattr(self, 'handle_' + type(frame.payload).__name__) - - meth(frame) +class Sender(object): + def __init__(self, channel_id, protocol): + self.channel_id = channel_id + self.protocol = protocol + + def send_method(self, method): + self.protocol.send_method(self.channel_id, method) + + +class FrameHandler(object): + def __init__(self, synchroniser, sender): + self.synchroniser = synchroniser + self.sender = sender + + def handle(self, frame): + try: + meth = getattr(self, 'handle_' + type(frame).__name__) + except AttributeError: + meth = getattr(self, 'handle_' + type(frame.payload).__name__) + + meth(frame) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 59280eb..7b607ba 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -1,126 +1,126 @@ -import asyncio -import collections -from . import frames -from . import spec - - -_TEST = False - - -class Dispatcher(object): - def __init__(self): - self.queue_writers = {} - self.closing = asyncio.Future() - - def add_writer(self, channel_id, writer): - self.queue_writers[channel_id] = writer - - def remove_writer(self, channel_id): - del self.queue_writers[channel_id] - - def dispatch(self, frame): - if isinstance(frame, frames.HeartbeatFrame): - return - if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): - return - writer = self.queue_writers[frame.channel_id] - writer.enqueue(frame) - - -class Synchroniser(object): - def __init__(self): - self._futures = OrderedManyToManyMap() - - def await(self, *expected_methods): - fut = asyncio.Future() - self._futures.add_item(expected_methods, fut) - return fut - - def notify(self, method, result=None): - fut = self._futures.get_leftmost(method) - fut.set_result(result) - self._futures.remove_item(fut) - - -def create_reader_and_writer(handler): - q = asyncio.Queue() - reader = QueueReader(handler, q) - writer = QueueWriter(q) - return reader, writer - - -# When ready() is called, wait for a frame to arrive on the queue. -# When the frame does arrive, dispatch it to the handler and do nothing -# until someone calls ready() again. -class QueueReader(object): - def __init__(self, handler, q): - self.handler = handler - self.q = q - self.is_waiting = False - - def ready(self): - assert not self.is_waiting, "ready() got called while waiting for a frame to be read" - self.is_waiting = True - t = asyncio.async(self._read_next()) - if _TEST: # this feels hacky to me - t._log_destroy_pending = False - - @asyncio.coroutine - def _read_next(self): - assert self.is_waiting, "a frame got read without ready() having been called" - frame = yield from self.q.get() - self.is_waiting = False - self.handler.handle(frame) - - -class QueueWriter(object): - def __init__(self, q): - self.q = q - - def enqueue(self, frame): - self.q.put_nowait(frame) - - -class OrderedManyToManyMap(object): - def __init__(self): - self._items = collections.defaultdict(OrderedSet) - - def add_item(self, keys, item): - for key in keys: - self._items[key].add(item) - - def remove_item(self, item): - for ordered_set in self._items.values(): - ordered_set.discard(item) - - def get_leftmost(self, key): - return self._items[key].first() - - -class OrderedSet(collections.MutableSet): - def __init__(self): - self._map = collections.OrderedDict() - - def __contains__(self, item): - return item in self._map - - def __iter__(self): - return iter(self._map.keys()) - - def __getitem__(self, ix): - return - - def __len__(self): - return len(self._map) - - def add(self, item): - self._map[item] = None - - def discard(self, item): - try: - del self._map[item] - except KeyError: - pass - - def first(self): - return next(iter(self._map)) +import asyncio +import collections +from . import frames +from . import spec + + +_TEST = False + + +class Dispatcher(object): + def __init__(self): + self.queue_writers = {} + self.closing = asyncio.Future() + + def add_writer(self, channel_id, writer): + self.queue_writers[channel_id] = writer + + def remove_writer(self, channel_id): + del self.queue_writers[channel_id] + + def dispatch(self, frame): + if isinstance(frame, frames.HeartbeatFrame): + return + if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): + return + writer = self.queue_writers[frame.channel_id] + writer.enqueue(frame) + + +class Synchroniser(object): + def __init__(self): + self._futures = OrderedManyToManyMap() + + def await(self, *expected_methods): + fut = asyncio.Future() + self._futures.add_item(expected_methods, fut) + return fut + + def notify(self, method, result=None): + fut = self._futures.get_leftmost(method) + fut.set_result(result) + self._futures.remove_item(fut) + + +def create_reader_and_writer(handler): + q = asyncio.Queue() + reader = QueueReader(handler, q) + writer = QueueWriter(q) + return reader, writer + + +# When ready() is called, wait for a frame to arrive on the queue. +# When the frame does arrive, dispatch it to the handler and do nothing +# until someone calls ready() again. +class QueueReader(object): + def __init__(self, handler, q): + self.handler = handler + self.q = q + self.is_waiting = False + + def ready(self): + assert not self.is_waiting, "ready() got called while waiting for a frame to be read" + self.is_waiting = True + t = asyncio.async(self._read_next()) + if _TEST: # this feels hacky to me + t._log_destroy_pending = False + + @asyncio.coroutine + def _read_next(self): + assert self.is_waiting, "a frame got read without ready() having been called" + frame = yield from self.q.get() + self.is_waiting = False + self.handler.handle(frame) + + +class QueueWriter(object): + def __init__(self, q): + self.q = q + + def enqueue(self, frame): + self.q.put_nowait(frame) + + +class OrderedManyToManyMap(object): + def __init__(self): + self._items = collections.defaultdict(OrderedSet) + + def add_item(self, keys, item): + for key in keys: + self._items[key].add(item) + + def remove_item(self, item): + for ordered_set in self._items.values(): + ordered_set.discard(item) + + def get_leftmost(self, key): + return self._items[key].first() + + +class OrderedSet(collections.MutableSet): + def __init__(self): + self._map = collections.OrderedDict() + + def __contains__(self, item): + return item in self._map + + def __iter__(self): + return iter(self._map.keys()) + + def __getitem__(self, ix): + return + + def __len__(self): + return len(self._map) + + def add(self, item): + self._map[item] = None + + def discard(self, item): + try: + del self._map[item] + except KeyError: + pass + + def first(self): + return next(iter(self._map)) diff --git a/test/util.py b/test/util.py index 185395d..a8c5595 100644 --- a/test/util.py +++ b/test/util.py @@ -1,109 +1,109 @@ -import asyncio -from contextlib import contextmanager -from unittest import mock -import asynqp.frames -from asynqp import protocol - - -class MockServer(object): - def __init__(self, protocol, tick): - self.protocol = protocol - self.tick = tick - self.data = [] - - def send_bytes(self, b): - self.protocol.data_received(b) - self.tick() - - def send_frame(self, frame): - self.send_bytes(frame.serialise()) - - def send_method(self, channel_number, method): - frame = asynqp.frames.MethodFrame(channel_number, method) - self.send_frame(frame) - - def reset(self): - self.data = [] - - def should_have_received_frames(self, expected_frames, any_order=False): - results = (read(x) for x in self.data) - frames = [x for x in results if x is not None] - if any_order: - for frame in expected_frames: - assert frame in frames, "{} should have been in {}".format(frame, frames) - else: - expected_frames = tuple(expected_frames) - assert expected_frames in windows(frames, len(expected_frames)), "{} should have been in {}".format(expected_frames, frames) - - def should_have_received_methods(self, channel_number, methods, any_order=False): - frames = (asynqp.frames.MethodFrame(channel_number, m) for m in methods) - self.should_have_received_frames(frames, any_order) - - def should_have_received_frame(self, expected_frame): - self.should_have_received_frames([expected_frame], any_order=True) - - def should_have_received_method(self, channel_number, method): - self.should_have_received_methods(channel_number, [method], any_order=True) - - def should_not_have_received_method(self, channel_number, method): - results = (read(x) for x in self.data) - frames = [x for x in results if x is not None] - - frame = asynqp.frames.MethodFrame(channel_number, method) - assert frame not in frames, "{} should not have been in {}".format(frame, frames) - - def should_not_have_received_any(self): - assert not self.data, "{} should have been empty".format(self.data) - - def should_have_received_bytes(self, b): - assert b in self.data - - -def read(data): - if data == b'AMQP\x00\x00\x09\x01': - return - - result = protocol.FrameReader().read_frame(data) - if result is None: - return - return result[0] - - -def windows(l, size): - return zip(*[l[x:] for x in range(size)]) - - -class FakeTransport(object): - def __init__(self, server): - self.server = server - self.closed = False - - def write(self, data): - self.server.data.append(data) - - def close(self): - self.closed = True - - -def any(cls): - class _any(cls): - def __init__(self): - pass - - def __eq__(self, other): - return isinstance(other, cls) - return _any() - - -@contextmanager -def silence_expected_destroy_pending_log(expected_coro_name=''): - real_async = asyncio.async - - def async(*args, **kwargs): - t = real_async(*args, **kwargs) - if expected_coro_name in repr(t): - t._log_destroy_pending = False - return t - - with mock.patch.object(asyncio, 'async', async): - yield +import asyncio +from contextlib import contextmanager +from unittest import mock +import asynqp.frames +from asynqp import protocol + + +class MockServer(object): + def __init__(self, protocol, tick): + self.protocol = protocol + self.tick = tick + self.data = [] + + def send_bytes(self, b): + self.protocol.data_received(b) + self.tick() + + def send_frame(self, frame): + self.send_bytes(frame.serialise()) + + def send_method(self, channel_number, method): + frame = asynqp.frames.MethodFrame(channel_number, method) + self.send_frame(frame) + + def reset(self): + self.data = [] + + def should_have_received_frames(self, expected_frames, any_order=False): + results = (read(x) for x in self.data) + frames = [x for x in results if x is not None] + if any_order: + for frame in expected_frames: + assert frame in frames, "{} should have been in {}".format(frame, frames) + else: + expected_frames = tuple(expected_frames) + assert expected_frames in windows(frames, len(expected_frames)), "{} should have been in {}".format(expected_frames, frames) + + def should_have_received_methods(self, channel_number, methods, any_order=False): + frames = (asynqp.frames.MethodFrame(channel_number, m) for m in methods) + self.should_have_received_frames(frames, any_order) + + def should_have_received_frame(self, expected_frame): + self.should_have_received_frames([expected_frame], any_order=True) + + def should_have_received_method(self, channel_number, method): + self.should_have_received_methods(channel_number, [method], any_order=True) + + def should_not_have_received_method(self, channel_number, method): + results = (read(x) for x in self.data) + frames = [x for x in results if x is not None] + + frame = asynqp.frames.MethodFrame(channel_number, method) + assert frame not in frames, "{} should not have been in {}".format(frame, frames) + + def should_not_have_received_any(self): + assert not self.data, "{} should have been empty".format(self.data) + + def should_have_received_bytes(self, b): + assert b in self.data + + +def read(data): + if data == b'AMQP\x00\x00\x09\x01': + return + + result = protocol.FrameReader().read_frame(data) + if result is None: + return + return result[0] + + +def windows(l, size): + return zip(*[l[x:] for x in range(size)]) + + +class FakeTransport(object): + def __init__(self, server): + self.server = server + self.closed = False + + def write(self, data): + self.server.data.append(data) + + def close(self): + self.closed = True + + +def any(cls): + class _any(cls): + def __init__(self): + pass + + def __eq__(self, other): + return isinstance(other, cls) + return _any() + + +@contextmanager +def silence_expected_destroy_pending_log(expected_coro_name=''): + real_async = asyncio.async + + def async(*args, **kwargs): + t = real_async(*args, **kwargs) + if expected_coro_name in repr(t): + t._log_destroy_pending = False + return t + + with mock.patch.object(asyncio, 'async', async): + yield From 4d9cfef826b32fb1aee13ee7edc3f38705fed3e6 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 6 May 2015 18:57:51 +0100 Subject: [PATCH 003/118] Update ez_setup --- ez_setup.py | 255 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 157 insertions(+), 98 deletions(-) diff --git a/ez_setup.py b/ez_setup.py index 1b8fd95..30d5fd0 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,18 +1,11 @@ #!/usr/bin/env python -"""Bootstrap setuptools installation -To use setuptools in your package's setup.py, include this -file in the same directory and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -To require a specific version of setuptools, set a download -mirror, or use an alternate download directory, simply supply -the appropriate options to ``use_setuptools()``. +""" +Setuptools bootstrapping installer. -This file can also be run as a script to install or upgrade setuptools. +Run this script to install or upgrade setuptools. """ + import os import shutil import sys @@ -23,19 +16,29 @@ import platform import textwrap import contextlib +import warnings from distutils import log +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + try: from site import USER_SITE except ImportError: USER_SITE = None -DEFAULT_VERSION = "3.4.4" +DEFAULT_VERSION = "15.2" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" +DEFAULT_SAVE_DIR = os.curdir + def _python_cmd(*args): """ + Execute a command. + Return True if the command succeeded. """ args = (sys.executable,) + args @@ -43,6 +46,7 @@ def _python_cmd(*args): def _install(archive_filename, install_args=()): + """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') @@ -54,6 +58,7 @@ def _install(archive_filename, install_args=()): def _build_egg(egg, archive_filename, to_dir): + """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) @@ -64,28 +69,36 @@ def _build_egg(egg, archive_filename, to_dir): raise IOError('Could not build the egg.') -def get_zip_class(): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ - class ContextualZipFile(zipfile.ZipFile): - def __enter__(self): - return self - def __exit__(self, type, value, traceback): - self.close - return zipfile.ZipFile if hasattr(zipfile.ZipFile, '__exit__') else \ - ContextualZipFile +class ContextualZipFile(zipfile.ZipFile): + + """Supplement ZipFile class to support context manager for Python 2.6.""" + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __new__(cls, *args, **kwargs): + """Construct a ZipFile or ContextualZipFile as appropriate.""" + if hasattr(zipfile.ZipFile, '__exit__'): + return zipfile.ZipFile(*args, **kwargs) + return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): - # extracting the archive + """ + Unzip filename to a temporary directory, set to the cwd. + + The unzipped target is cleaned up after. + """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) - with get_zip_class()(filename) as archive: + with ContextualZipFile(filename) as archive: archive.extractall() # going in the directory @@ -100,6 +113,7 @@ def archive_context(filename): def _do_download(version, download_base, to_dir, download_delay): + """Download Setuptools.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): @@ -117,41 +131,77 @@ def _do_download(version, download_base, to_dir, download_delay): setuptools.bootstrap_install_from = egg -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, download_delay=15): + """ + Ensure that a setuptools version is installed. + + Return None. Raise SystemExit if the requested version + or later cannot be installed. + """ to_dir = os.path.abspath(to_dir) + + # prior to importing, capture the module state for + # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) + try: import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: pkg_resources.require("setuptools>=" + version) + # a suitable version is already installed return + except ImportError: + # pkg_resources not available; setuptools is not installed; download + pass except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, download_delay) + # no version of setuptools was found; allow download + pass except pkg_resources.VersionConflict as VC_err: if imported: - msg = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. + _conflict_bail(VC_err, version) + + # otherwise, unload pkg_resources to allow the downloaded version to + # take precedence. + del pkg_resources + _unload_pkg_resources() + + return _do_download(version, download_base, to_dir, download_delay) - (Currently using {VC_err.args[0]!r}) - """).format(VC_err=VC_err, version=version) - sys.stderr.write(msg) - sys.exit(2) - # otherwise, reload ok - del pkg_resources, sys.modules['pkg_resources'] - return _do_download(version, download_base, to_dir, download_delay) +def _conflict_bail(VC_err, version): + """ + Setuptools was imported prior to invocation, so it is + unsafe to unload it. Bail out. + """ + conflict_tmpl = textwrap.dedent(""" + The required version of setuptools (>={version}) is not available, + and can't be installed while this script is running. Please + install a more recent version first, using + 'easy_install -U setuptools'. + + (Currently using {VC_err.args[0]!r}) + """) + msg = conflict_tmpl.format(**locals()) + sys.stderr.write(msg) + sys.exit(2) + + +def _unload_pkg_resources(): + del_modules = [ + name for name in sys.modules + if name.startswith('pkg_resources') + ] + for mod_name in del_modules: + del sys.modules[mod_name] + def _clean_check(cmd, target): """ - Run the command to download target. If the command fails, clean up before - re-raising the error. + Run the command to download target. + + If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) @@ -160,115 +210,110 @@ def _clean_check(cmd, target): os.unlink(target) raise + def download_file_powershell(url, target): """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. + Download the file at url to target using Powershell. + + Powershell will validate trust. + Raise an exception if the command cannot complete. """ target = os.path.abspath(target) + ps_cmd = ( + "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " + "[System.Net.CredentialCache]::DefaultCredentials; " + "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" + % vars() + ) cmd = [ 'powershell', '-Command', - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), + ps_cmd, ] _clean_check(cmd, target) + def has_powershell(): + """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_powershell.viable = has_powershell + def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) + def has_curl(): cmd = ['curl', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_curl.viable = has_curl + def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) + def has_wget(): cmd = ['wget', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_wget.viable = has_wget + def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - src = dst = None + """Use Python to download the file, without connection authentication.""" + src = urlopen(url) try: - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. + # Read all the data in one block. data = src.read() - dst = open(target, "wb") - dst.write(data) finally: - if src: - src.close() - if dst: - dst.close() + src.close() + # Write all the data in one block to avoid creating a partial file. + with open(target, "wb") as dst: + dst.write(data) download_file_insecure.viable = lambda: True + def get_best_downloader(): - downloaders = [ + downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, - ] + ) + viable_downloaders = (dl for dl in downloaders if dl.viable()) + return next(viable_downloaders, None) - for dl in downloaders: - if dl.viable(): - return dl -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, delay=15, + downloader_factory=get_best_downloader): """ - Download setuptools from a specified location and return its filename + Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end + as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. @@ -287,16 +332,18 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, downloader(url, saveto) return os.path.realpath(saveto) + def _build_install_args(options): """ - Build the arguments to 'python setup.py install' on the setuptools package + Build the arguments to 'python setup.py install' on the setuptools package. + + Returns list of command line arguments. """ return ['--user'] if options.user_install else [] + def _parse_args(): - """ - Parse the command line for options - """ + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, @@ -314,18 +361,30 @@ def _parse_args(): '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) + parser.add_option( + '--to-dir', + help="Directory to save (and re-use) package", + default=DEFAULT_SAVE_DIR, + ) options, args = parser.parse_args() # positional arguments are ignored return options -def main(): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - archive = download_setuptools( + +def _download_args(options): + """Return args for download_setuptools function from cmdline args.""" + return dict( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, + to_dir=options.to_dir, ) + + +def main(): + """Install or upgrade setuptools and EasyInstall.""" + options = _parse_args() + archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': From 0795309c38aac3e3742177a93673408569bec947 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 11 May 2015 13:11:33 +0100 Subject: [PATCH 004/118] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 182eae9..baa3bf4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,10 @@ To contribute to this project, submit a [pull request](https://help.github.com/a 3. [Open a pull request](https://help.github.com/articles/creating-a-pull-request) in this repo to merge your topic branch into the mainstream 4. I'll review your changes and merge your pull request as soon as possible +If you want to contribute to the project, but are not sure what you want to work on, +I am always happy for help on any of the [open issues](https://github.com/benjamin-hodgson/asynqp/issues) +in the GitHub tracker. + This project is built using Test-Driven-Development. So if you're planning to contribute a feature or bugfix, please **ensure that it is covered by tests** before submitting it for review. Use your best judgment to From 7bf36738bdafa426d48112a4366f89df9dd0df6d Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 11:39:22 +0100 Subject: [PATCH 005/118] Run flake8 on travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8200de4..488a61a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,11 @@ python: 3.4 services: rabbitmq install: - - pip install contexts colorama sphinx + - pip install contexts colorama sphinx flake8 - python setup.py install script: + - flake8 src test --ignore=E501 - run-contexts -v - cd doc && make html From 8827818171f451aa5086430ac0f5fa1c423272a8 Mon Sep 17 00:00:00 2001 From: Krzysztof Urbaniak Date: Thu, 7 May 2015 00:32:35 +0200 Subject: [PATCH 006/118] do not pass host and port to loop.create_connection when there's a sock param --- src/asynqp/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index 8577051..bf52eb3 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -35,8 +35,12 @@ def connect(host='localhost', loop = asyncio.get_event_loop() if loop is None else loop + if 'sock' not in kwargs: + kwargs['host'] = host + kwargs['port'] = port + dispatcher = Dispatcher() - transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), host=host, port=port, **kwargs) + transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) connection = yield from open_connection(loop, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) return connection From 50cd0e75f0e9247bbead78955322ccf97673b330 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 12:04:53 +0100 Subject: [PATCH 007/118] add integration test for pull request #14 from @urbaniak --- test/integration_tests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/integration_tests.py b/test/integration_tests.py index 80f581c..7e2f2e7 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -1,5 +1,6 @@ import asyncio import asynqp +import socket import contexts @@ -54,6 +55,22 @@ def cleanup_the_connection(self): self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) +class WhenConnectingToRabbitWithAnExistingSocket: + def given_the_loop(self): + self.loop = asyncio.get_event_loop() + self.sock = socket.create_connection(("localhost", 5672)) + + def when_I_connect(self): + self.connection = self.loop.run_until_complete(asyncio.wait_for(asynqp.connect(sock=self.sock), 0.2)) + + def it_should_connect(self): + assert self.connection is not None + + def cleanup_the_connection(self): + self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) + self.sock.close() + + class WhenOpeningAChannel(ConnectionContext): def when_I_open_a_channel(self): self.channel = self.loop.run_until_complete(asyncio.wait_for(self.connection.open_channel(), 0.2)) From b6441a99966cb88baedff03b41a2315b6605781d Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 17:46:25 +0100 Subject: [PATCH 008/118] __all__ --- src/asynqp/__init__.py | 19 +++++++++++++------ src/asynqp/exceptions.py | 7 +++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index bf52eb3..1bc50c9 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,10 +1,17 @@ import asyncio -from .exceptions import AMQPError, Deleted # noqa -from .message import Message, IncomingMessage # noqa -from .connection import Connection # noqa -from .channel import Channel # noqa -from .exchange import Exchange # noqa -from .queue import Queue, QueueBinding, Consumer # noqa +from .exceptions import AMQPError, UndeliverableMessage, Deleted +from .message import Message, IncomingMessage +from .connection import Connection +from .channel import Channel +from .exchange import Exchange +from .queue import Queue, QueueBinding, Consumer + + +__all__ = [ + "AMQPError", "UndeliverableMessage", "Deleted", + "Message", "IncomingMessage", + "Connection", "Channel", "Exchange", "Queue", "QueueBinding", "Consumer" +] @asyncio.coroutine diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 7e942e1..3ab2570 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -1,3 +1,10 @@ +__all__ = [ + "AMQPError", + "UndeliverableMessage", + "Deleted" +] + + class AMQPError(IOError): pass From 067c6516606f0cc318a7a586c7bba19dfc794860 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 17:46:47 +0100 Subject: [PATCH 009/118] Expose transport and protocol on Connection --- src/asynqp/__init__.py | 2 +- src/asynqp/connection.py | 16 +++++++++++++--- test/base_contexts.py | 4 ++-- test/connection_tests.py | 4 ++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index 1bc50c9..bcd71ab 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -49,7 +49,7 @@ def connect(host='localhost', dispatcher = Dispatcher() transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) - connection = yield from open_connection(loop, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) + connection = yield from open_connection(loop, transport, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) return connection diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 4e4dc31..7d952ee 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -30,8 +30,18 @@ class Connection(object): .. attribute:: closed a :class:`~asyncio.Future` which is done when the handshake to close the connection has finished + + .. attribute:: transport + + The :class:`~asyncio.BaseTransport` over which the connection is communicating with the server + + .. attribute:: protocol + + The :class:`~asyncio.Protocol` which is paired with the transport """ - def __init__(self, loop, protocol, synchroniser, sender, dispatcher, connection_info): + def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): + self.transport = transport + self.protocol = protocol self.synchroniser = synchroniser self.sender = sender self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) @@ -69,11 +79,11 @@ def close(self): @asyncio.coroutine -def open_connection(loop, protocol, dispatcher, connection_info): +def open_connection(loop, transport, protocol, dispatcher, connection_info): synchroniser = routing.Synchroniser() sender = ConnectionMethodSender(protocol) - connection = Connection(loop, protocol, synchroniser, sender, dispatcher, connection_info) + connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) handler = ConnectionFrameHandler(synchroniser, sender, protocol, connection) reader, writer = routing.create_reader_and_writer(handler) diff --git a/test/base_contexts.py b/test/base_contexts.py index 923c3a5..30342f2 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -43,7 +43,7 @@ def given_a_mock_server_on_the_other_end_of_the_transport(self): class OpenConnectionContext(MockServerContext): def given_an_open_connection(self): connection_info = ConnectionInfo('guest', 'guest', '/') - task = asyncio.async(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + task = asyncio.async(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.tick() start_method = spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US') @@ -126,7 +126,7 @@ def given_the_pieces_i_need_for_a_connection(self): class LegacyOpenConnectionContext(LegacyConnectionContext): def given_an_open_connection(self): - task = asyncio.async(open_connection(self.loop, self.protocol, self.dispatcher, self.connection_info)) + task = asyncio.async(open_connection(self.loop, self.protocol.transport, self.protocol, self.dispatcher, self.connection_info)) self.tick() start_frame = asynqp.frames.MethodFrame(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) diff --git a/test/connection_tests.py b/test/connection_tests.py index a55b785..87613ba 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -10,7 +10,7 @@ class WhenRespondingToConnectionStart(MockServerContext): def given_I_wrote_the_protocol_header(self): connection_info = ConnectionInfo('guest', 'guest', '/') - self.async_partial(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) def when_ConnectionStart_arrives(self): self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) @@ -28,7 +28,7 @@ def it_should_send_start_ok(self): class WhenRespondingToConnectionTune(MockServerContext): def given_a_started_connection(self): connection_info = ConnectionInfo('guest', 'guest', '/') - self.async_partial(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) def when_ConnectionTune_arrives(self): From c848bf35673c7315ec3d76d8e2858848f3402ba8 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 18:08:03 +0100 Subject: [PATCH 010/118] address #12 by allowing properties to be set with setattr --- src/asynqp/message.py | 17 ++++++++++++----- test/message_tests.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/asynqp/message.py b/src/asynqp/message.py index ce86ae2..15722ac 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -75,23 +75,30 @@ def __init__(self, body, *, timestamp = timestamp if timestamp is not None else datetime.now() - self.properties = OrderedDict() + self._properties = OrderedDict() for name, amqptype in self.property_types.items(): value = locals()[name] if value is not None: value = amqptype(value) - self.properties[name] = value + self._properties[name] = value def __eq__(self, other): return (self.body == other.body - and self.properties == other.properties) + and self._properties == other._properties) def __getattr__(self, name): try: - return self.properties[name] + return self._properties[name] except KeyError as e: raise AttributeError from e + def __setattr__(self, name, value): + amqptype = self.property_types.get(name) + if amqptype is not None: + self._properties[name] = value if isinstance(value, amqptype) else amqptype(value) + return + super().__setattr__(name, value) + def json(self): """ Parse the message body as JSON. @@ -143,7 +150,7 @@ def reject(self, *, requeue=True): def get_header_payload(message, class_id): - return ContentHeaderPayload(class_id, len(message.body), list(message.properties.values())) + return ContentHeaderPayload(class_id, len(message.body), list(message._properties.values())) # NB: the total frame size will be 8 bytes larger than frame_body_size diff --git a/test/message_tests.py b/test/message_tests.py index 48223ef..21d914d 100644 --- a/test/message_tests.py +++ b/test/message_tests.py @@ -3,6 +3,7 @@ import uuid from datetime import datetime import asynqp +from asynqp import amqptypes from asynqp import message from asynqp import spec from asynqp import frames @@ -184,3 +185,38 @@ def when_I_get_the_json(self): def it_should_give_me_the_body(self): assert self.result == self.body + + +class WhenSettingAProperty: + def given_a_message(self): + self.msg = asynqp.Message("abc") + + def when_I_set_a_property(self): + self.msg.content_type = "application/json" + + def it_should_cast_it_to_the_correct_amqp_type(self): + assert isinstance(self.msg.content_type, amqptypes.ShortStr) + assert self.msg.content_type == amqptypes.ShortStr("application/json") + + +class WhenSettingAPropertyAndIHaveAlreadyCastItMyself: + def given_a_message(self): + self.msg = asynqp.Message("abc") + self.val = amqptypes.ShortStr("application/json") + + def when_I_set_a_property(self): + self.msg.content_type = self.val + + def it_should_not_attempt_to_cast_it(self): + assert self.msg.content_type is self.val + + +class WhenSettingAnAttributeThatIsNotAProperty: + def given_a_message(self): + self.msg = asynqp.Message("abc") + + def when_I_set_a_property(self): + self.msg.foo = 123 + + def it_should_not_attempt_to_cast_it(self): + assert self.msg.foo == 123 From c8d46e8f8c4f34f980ea91fb80d49b93f9f1e96c Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 12 May 2015 18:12:00 +0100 Subject: [PATCH 011/118] improve docs of Message --- src/asynqp/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/asynqp/message.py b/src/asynqp/message.py index 15722ac..1881bca 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -19,7 +19,9 @@ class Message(object): dicts will be converted to a string using JSON. :param dict headers: a dictionary of message headers :param str content_type: MIME content type - :param str content_encoding: MIME encoding + (defaults to 'application/json' if :code:`body` is a :class:`dict`, + or 'application/octet-stream' otherwise) + :param str content_encoding: MIME encoding (defaults to 'utf-8') :param int delivery_mode: 1 for non-persistent, 2 for persistent :param int priority: message priority - integer between 0 and 9 :param str correlation_id: correlation id of the message *(for applications)* From ca15931a0d958d04fff38cdbde6bcd9d0b89c743 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Thu, 14 May 2015 10:15:25 +0100 Subject: [PATCH 012/118] fix __all__ --- src/asynqp/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index bcd71ab..c687ac1 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -10,7 +10,8 @@ __all__ = [ "AMQPError", "UndeliverableMessage", "Deleted", "Message", "IncomingMessage", - "Connection", "Channel", "Exchange", "Queue", "QueueBinding", "Consumer" + "Connection", "Channel", "Exchange", "Queue", "QueueBinding", "Consumer", + "connect", "connect_and_open_channel" ] From 8d0e45f43c86d4595fb7f98e6906e2cca27e3ba5 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 18 May 2015 20:58:20 +0100 Subject: [PATCH 013/118] Copy fix from readme into docs front page --- doc/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.rst b/doc/index.rst index 3882b05..1f91134 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -42,6 +42,7 @@ Example # Acknowledge a delivered message received_message.ack() + yield from channel.close() yield from connection.close() From f665308473fb84d03b5a5084d18d1443fea376af Mon Sep 17 00:00:00 2001 From: Mike Lenzen Date: Mon, 18 May 2015 16:16:23 +0600 Subject: [PATCH 014/118] Added new exceptions and logic to allow for designing programs that automatically reconnect on a lost connection Added ConnectionClosedError and ConnectionLostError. ConnectionClosedError is raised when the connection is closed normally, ConnectionLostError is raised when the connection is lost unexpectedly. cleanup Send poison pill when the connection is closed so that cleanup methods (Channel.close, Connection.close, Consumer.cancel) do not block waiting for a message from the server on a closed connection. Fixes from experience with an actual example script consistently handle setting futures inside syncronizer when poison pill is swallowed --- src/asynqp/__init__.py | 8 ++- src/asynqp/bases.py | 3 + src/asynqp/exceptions.py | 17 ++++++ src/asynqp/frames.py | 8 +++ src/asynqp/protocol.py | 15 ++++- src/asynqp/routing.py | 42 ++++++++++++++ test/__init__.py | 6 ++ test/channel_tests.py | 4 +- test/heartbeat_tests.py | 13 +++++ test/integration_tests.py | 113 ++++++++++++++++++++++++++++++++++++++ test/protocol_tests.py | 48 ++++++++++++++++ test/queue_tests.py | 3 +- test/util.py | 13 +++++ 13 files changed, 288 insertions(+), 5 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index c687ac1..c969f41 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -48,7 +48,13 @@ def connect(host='localhost', kwargs['port'] = port dispatcher = Dispatcher() - transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) + try: + transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) + except (ConnectionRefusedError, OSError): + # Throw a single exception instead of two + raise ConnectionRefusedError('Failed to connect - host: {} port: {}' + ' username: {} password: {} virtual_host: {}' + .format(host, port, username, password, virtual_host)) connection = yield from open_connection(loop, transport, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) return connection diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py index 22dec0c..ca89e22 100644 --- a/src/asynqp/bases.py +++ b/src/asynqp/bases.py @@ -19,3 +19,6 @@ def handle(self, frame): meth = getattr(self, 'handle_' + type(frame.payload).__name__) meth(frame) + + def handle_ConnectionClosedPoisonPillFrame(self, frame): + self.synchroniser.notify_connection_closed() diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 3ab2570..0372751 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -1,5 +1,7 @@ __all__ = [ "AMQPError", + "ConnectionClosedError", + "ConnectionLostError", "UndeliverableMessage", "Deleted" ] @@ -9,6 +11,21 @@ class AMQPError(IOError): pass +class ConnectionClosedError(ConnectionError): + ''' + Connection was closed normally by either the amqp server + or the client. + ''' + pass + + +class ConnectionLostError(ConnectionClosedError): + ''' + Connection was closed unexpectedly + ''' + pass + + class UndeliverableMessage(ValueError): pass diff --git a/src/asynqp/frames.py b/src/asynqp/frames.py index 70d5055..72e3d47 100644 --- a/src/asynqp/frames.py +++ b/src/asynqp/frames.py @@ -64,3 +64,11 @@ class HeartbeatFrame(Frame): def __init__(self): pass + + +class ConnectionClosedPoisonPillFrame(Frame): + channel_id = 0 + payload = b'' + + def __init__(self): + pass diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 7fcf918..d1cb260 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -2,7 +2,7 @@ import struct from . import spec from . import frames -from .exceptions import AMQPError +from .exceptions import AMQPError, ConnectionLostError, ConnectionClosedError class AMQP(asyncio.Protocol): @@ -47,6 +47,18 @@ def send_protocol_header(self): def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) + def connection_lost(self, exc): + self._send_connection_closed_poison_pill() + if exc is None: + raise ConnectionClosedError('The connection was closed') + else: + raise ConnectionLostError('The connection was unexpectedly lost') + + def _send_connection_closed_poison_pill(self): + frame = frames.ConnectionClosedPoisonPillFrame() + # send the poison pill to all open queues + self.dispatcher.dispatch_all(frame) + class FrameReader(object): def __init__(self): @@ -108,3 +120,4 @@ def heartbeat_received(self): def heartbeat_timed_out(self): self.protocol.send_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + self.protocol.connection_lost(ConnectionLostError) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 7b607ba..4427d27 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -26,13 +26,31 @@ def dispatch(self, frame): writer = self.queue_writers[frame.channel_id] writer.enqueue(frame) + def dispatch_all(self, frame): + for writer in self.queue_writers.values(): + writer.enqueue(frame) + class Synchroniser(object): + _blocking_methods = set((spec.BasicCancelOK, # Consumer.cancel + spec.ChannelCloseOK, # Channel.close + spec.ConnectionCloseOK)) # Connection.close + def __init__(self): self._futures = OrderedManyToManyMap() + self.connection_closed = False def await(self, *expected_methods): fut = asyncio.Future() + + if self.connection_closed: + for method in expected_methods: + if method in self._blocking_methods and not fut.done(): + fut.set_result(None) + if not fut.done(): + fut.set_exception(ConnectionError) + return fut + self._futures.add_item(expected_methods, fut) return fut @@ -41,6 +59,27 @@ def notify(self, method, result=None): fut.set_result(result) self._futures.remove_item(fut) + def notify_connection_closed(self): + self.connection_closed = True + # Give a proper notification to blocking methods + for method in self._blocking_methods: + while True: + try: + self.notify(method) + except StopIteration: + break + + # Set an exception for all others + for method in self._futures.keys(): + if method not in self._blocking_methods: + while True: + try: + fut = self._futures.get_leftmost(method) + fut.set_exception(ConnectionError) + self._futures.remove_item(fut) + except StopIteration: + break + def create_reader_and_writer(handler): q = asyncio.Queue() @@ -96,6 +135,9 @@ def remove_item(self, item): def get_leftmost(self, key): return self._items[key].first() + def keys(self): + return self._items.keys() + class OrderedSet(collections.MutableSet): def __init__(self): diff --git a/test/__init__.py b/test/__init__.py index e69de29..5dbd8ac 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -0,0 +1,6 @@ +import asyncio +from .util import testing_exception_handler + + +loop = asyncio.get_event_loop() +loop.set_exception_handler(testing_exception_handler) diff --git a/test/channel_tests.py b/test/channel_tests.py index 8447df3..81ea08c 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -174,7 +174,7 @@ def it_should_set_the_reply_code(self): assert self.exception.args == (self.expected_message,) def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(util.testing_exception_handler) # test that the call to handler.ready() happens at the correct time @@ -195,7 +195,7 @@ def it_should_throw_the_exception_again(self): assert self.exception is not None def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(util.testing_exception_handler) def return_msg(self): method = spec.BasicReturn(123, "you messed up", "the.exchange", "the.routing.key") diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index a97c0b1..59631ae 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -2,6 +2,7 @@ from asynqp import spec from asynqp import frames from asynqp import protocol +from asynqp.exceptions import ConnectionLostError from .base_contexts import ProtocolContext, MockLoopContext @@ -69,3 +70,15 @@ def when_the_heartbeat_times_out(self): def it_should_send_connection_close(self): self.protocol.send_method.assert_called_once_with(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + + +class WhenTheHeartbeatTimesOutCallProtocolConnectionLost(MockLoopContext): + def given_a_hearbeat_monitor(self): + self.protocol = mock.Mock(spec=protocol.AMQP) + self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) + + def when_the_heartbeat_times_out(self): + self.heartbeat_monitor.heartbeat_timed_out() + + def it_should_call_protocol_lost_connection(self): + self.protocol.connection_lost.assert_called_once_with(ConnectionLostError) diff --git a/test/integration_tests.py b/test/integration_tests.py index 7e2f2e7..15153bf 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -2,6 +2,7 @@ import asynqp import socket import contexts +from .util import testing_exception_handler class ConnectionContext: @@ -216,3 +217,115 @@ def when_I_cancel_the_consumer_and_also_get_a_message(self): def it_should_not_throw(self): assert self.exception is None + + +class WhenAConnectionIsClosed: + def given_an_exception_handler_and_connection(self): + self.loop = asyncio.get_event_loop() + self.connection_closed_error_raised = False + self.loop.set_exception_handler(self.exception_handler) + self.connection = self.loop.run_until_complete(asynqp.connect()) + + def exception_handler(self, loop, context): + exception = context.get('exception') + if type(exception) is asynqp.exceptions.ConnectionClosedError: + self.connection_closed_error_raised = True + else: + self.loop.default_exception_handler(context) + + def when_the_connection_is_closed(self): + self.loop.run_until_complete(self.connection.close()) + + def it_should_raise_a_connection_closed_error(self): + assert self.connection_closed_error_raised is True + + def cleanup(self): + self.loop.set_exception_handler(testing_exception_handler) + + +class WhenAConnectionIsLost: + def given_an_exception_handler_and_connection(self): + self.loop = asyncio.get_event_loop() + self.connection_lost_error_raised = False + self.loop.set_exception_handler(self.exception_handler) + self.connection = self.loop.run_until_complete(asynqp.connect()) + + def exception_handler(self, loop, context): + exception = context.get('exception') + if type(exception) is asynqp.exceptions.ConnectionLostError: + self.connection_lost_error_raised = True + self.loop.stop() + else: + self.loop.default_exception_handler(context) + + def when_the_heartbeat_times_out(self): + self.loop.call_soon(self.connection + .protocol + .heartbeat_monitor.heartbeat_timed_out) + self.loop.run_forever() + + def it_should_raise_a_connection_closed_error(self): + assert self.connection_lost_error_raised is True + + def cleanup(self): + self.loop.set_exception_handler(testing_exception_handler) + + +class WhenAConnectionIsClosedCloseConnection: + def given_a_connection(self): + self.loop = asyncio.get_event_loop() + self.connection = self.loop.run_until_complete(asynqp.connect()) + + def when_connection_is_closed(self): + self.connection.transport.close() + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) + + +class WhenAConnectionIsClosedCloseChannel: + def given_a_channel(self): + self.loop = asyncio.get_event_loop() + self.connection = self.loop.run_until_complete(asynqp.connect()) + self.channel = self.loop.run_until_complete(self.connection.open_channel()) + + def when_connection_is_closed(self): + self.connection.transport.close() + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + +class WhenAConnectionIsClosedCancelConsuming: + def given_a_consumer(self): + asynqp.routing._TEST = True + self.loop = asyncio.get_event_loop() + self.connection = self.loop.run_until_complete(asynqp.connect()) + self.channel = self.loop.run_until_complete(self.connection.open_channel()) + self.exchange = self.loop.run_until_complete( + self.channel.declare_exchange(name='name', + type='direct', + durable=False, + auto_delete=True)) + + self.queue = self.loop.run_until_complete( + self.channel.declare_queue(name='', + durable=False, + exclusive=True, + auto_delete=True)) + + self.loop.run_until_complete(self.queue.bind(self.exchange, + 'name')) + + self.consumer = self.loop.run_until_complete( + self.queue.consume(lambda x: x, exclusive=True) + ) + + def when_connection_is_closed(self): + self.connection.transport.close() + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) + + def cleanup(self): + asynqp.routing._TEST = False diff --git a/test/protocol_tests.py b/test/protocol_tests.py index 841843d..fb5c35e 100644 --- a/test/protocol_tests.py +++ b/test/protocol_tests.py @@ -3,7 +3,9 @@ import asynqp from asynqp import spec from asynqp import protocol +from asynqp.exceptions import ConnectionClosedError, ConnectionLostError from .base_contexts import MockDispatcherContext, MockServerContext +from .util import testing_exception_handler class WhenInitiatingProceedings(MockServerContext): @@ -130,3 +132,49 @@ def because_two_frames_arrive_in_bits(self, fragments): def it_should_dispatch_the_method_twice(self): self.dispatcher.dispatch.assert_has_calls([mock.call(self.expected_frame), mock.call(self.expected_frame)]) + + +class WhenTheConnectionIsClosed(MockServerContext): + def given_an_exception_handler(self): + self.connection_closed_error_raised = False + self.loop.set_exception_handler(self.exception_handler) + + def exception_handler(self, loop, context): + exception = context.get('exception') + if type(exception) is ConnectionClosedError: + self.connection_closed_error_raised = True + else: + self.loop.default_exception_handler(context) + + def when_the_connection_is_closed(self): + self.loop.call_soon(self.protocol.connection_lost, None) + self.tick() + + def it_should_raise_a_connection_closed_error(self): + assert self.connection_closed_error_raised is True + + def cleanup(self): + self.loop.set_exception_handler(testing_exception_handler) + + +class WhenTheConnectionIsLost(MockServerContext): + def given_an_exception_handler(self): + self.connection_lost_error_raised = False + self.loop.set_exception_handler(self.exception_handler) + + def exception_handler(self, loop, context): + exception = context.get('exception') + if type(exception) is ConnectionLostError: + self.connection_lost_error_raised = True + else: + self.loop.default_exception_handler(context) + + def when_the_connection_is_closed(self): + self.loop.call_soon(self.protocol.connection_lost, Exception) + self.tick() + + def it_should_raise_a_connection_lost_error(self): + assert self.connection_lost_error_raised is True + + def cleanup(self): + self.loop.set_exception_handler(testing_exception_handler) diff --git a/test/queue_tests.py b/test/queue_tests.py index 06d5bda..dd3ad45 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -5,6 +5,7 @@ from asynqp import frames from asynqp import spec from .base_contexts import OpenChannelContext, QueueContext, ExchangeContext, BoundQueueContext, ConsumerContext +from .util import testing_exception_handler class WhenDeclaringAQueue(OpenChannelContext): @@ -243,7 +244,7 @@ def deliver_msg(self): self.tick() def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(testing_exception_handler) class WhenICancelAConsumer(ConsumerContext): diff --git a/test/util.py b/test/util.py index a8c5595..b401766 100644 --- a/test/util.py +++ b/test/util.py @@ -3,6 +3,19 @@ from unittest import mock import asynqp.frames from asynqp import protocol +from asynqp.exceptions import ConnectionClosedError + + +def testing_exception_handler(loop, context): + ''' + Hides the expected ``ConnectionClosedErrors`` and + ``ConnectionLostErros`` during tests + ''' + exception = context.get('exception') + if exception and isinstance(exception, ConnectionClosedError): + pass + else: + loop.default_exception_handler(context) class MockServer(object): From eb78250d896e59836e24e4b2028c42a4b42ee3f3 Mon Sep 17 00:00:00 2001 From: Mike Lenzen Date: Tue, 19 May 2015 10:21:31 +0600 Subject: [PATCH 015/118] Example script --- examples/reconnecting.py | 157 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 examples/reconnecting.py diff --git a/examples/reconnecting.py b/examples/reconnecting.py new file mode 100644 index 0000000..a8b257a --- /dev/null +++ b/examples/reconnecting.py @@ -0,0 +1,157 @@ +''' +Example async consumer and publisher that will reconnect +automatically when a connection to rabbitmq is broken and +restored. + + +.. note:: + + No attempt is made to re-send messages that are generated + while the connection is down. +''' +import asyncio +import asynqp +from asyncio.futures import InvalidStateError + + +# Global variables are ugly, but this is a simple example +CHANNELS = [] +CONNECTION = None +CONSUMER = None +PRODUCER = None + + +@asyncio.coroutine +def setup_connection(loop): + # connect to the RabbitMQ broker + connection = yield from asynqp.connect('localhost', + 5672, + username='guest', + password='guest') + + return connection + + +@asyncio.coroutine +def setup_queue_and_exchange(connection): + # Open a communications channel + channel = yield from connection.open_channel() + + # Create a queue and an exchange on the broker + exchange = yield from channel.declare_exchange('test.exchange', 'direct') + queue = yield from channel.declare_queue('test.queue') + + # Save a reference to each channel so we can close it later + CHANNELS.append(channel) + + # Bind the queue to the exchange, so the queue will get messages published to the exchange + yield from queue.bind(exchange, 'routing.key') + + return exchange, queue + + +@asyncio.coroutine +def setup_consumer(connection): + # callback will be called each time a message is received from the queue + def callback(msg): + print('Received: {}'.format(msg.body)) + msg.ack() + + _, queue = yield from setup_queue_and_exchange(connection) + + # connect the callback to the queue + consumer = yield from queue.consume(callback) + return consumer + + +@asyncio.coroutine +def setup_producer(connection): + ''' + The producer will live as an asyncio.Task + to stop it call Task.cancel() + ''' + exchange, _ = yield from setup_queue_and_exchange(connection) + + count = 0 + while True: + msg = asynqp.Message('Message #{}'.format(count)) + exchange.publish(msg, 'routing.key') + yield from asyncio.sleep(1) + count += 1 + + +@asyncio.coroutine +def start(loop): + ''' + Creates a connection, starts the consumer and producer. + If it fails to reconnect, it waits 1 second then try again + ''' + global CONNECTION + global CONSUMER + global PRODUCER + try: + CONNECTION = yield from setup_connection(loop) + CONSUMER = yield from setup_consumer(CONNECTION) + PRODUCER = loop.create_task(setup_producer(CONNECTION)) + except ConnectionError: + print('failed to connect, trying again.') + yield from asyncio.sleep(1) + loop.create_task(start(loop)) + + +@asyncio.coroutine +def stop(): + ''' + Cleans up connections, channels, consumers and producers + when the connection is closed. + ''' + global CHANNELS + global CONNECTION + global PRODUCER + global CONSUMER + + yield from CONSUMER.cancel() # this is a coroutine + PRODUCER.cancel() # this is not + + for channel in CHANNELS: + yield from channel.close() + CHANNELS = [] + + if CONNECTION is not None: + try: + yield from CONNECTION.close() + except InvalidStateError: + pass # should be automatically closed, so this is expected + CONNECTION = None + + +def connection_lost_handler(loop, context): + ''' + Here we setup a custom exception handler to listen for + ConnectionErrors. + + The exceptions we can catch follow this inheritance scheme + | + - ConnectionError - base + | + - asynqp.exceptions.ConnectionClosedError - connection closed properly + | + - asynqp.exceptions.ConnectionLostError - closed unexpectedly + ''' + exception = context.get('exception') + if isinstance(exception, asynqp.exceptions.ConnectionClosedError): + print('Connection lost -- trying to reconnect') + # close everything before recpnnecting + close_task = loop.create_task(stop()) + asyncio.wait_for(close_task, None) + # reconnect + loop.create_task(start(loop)) + else: + # default behaviour + loop.default_exception_handler(context) + + +loop = asyncio.get_event_loop() +loop.set_exception_handler(connection_lost_handler) +loop.create_task(start(loop)) +loop.run_forever() From 4296074b4af510f1b77fab69d777b9cdb92eae52 Mon Sep 17 00:00:00 2001 From: Mike Lenzen Date: Wed, 20 May 2015 22:51:55 +0600 Subject: [PATCH 016/118] Comment updates --- src/asynqp/__init__.py | 4 ++-- src/asynqp/protocol.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index c969f41..f4f02de 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -50,11 +50,11 @@ def connect(host='localhost', dispatcher = Dispatcher() try: transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) - except (ConnectionRefusedError, OSError): + except (ConnectionRefusedError, OSError) as e: # Throw a single exception instead of two raise ConnectionRefusedError('Failed to connect - host: {} port: {}' ' username: {} password: {} virtual_host: {}' - .format(host, port, username, password, virtual_host)) + .format(host, port, username, password, virtual_host)) from e connection = yield from open_connection(loop, transport, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) return connection diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index d1cb260..6ae1c35 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -52,7 +52,7 @@ def connection_lost(self, exc): if exc is None: raise ConnectionClosedError('The connection was closed') else: - raise ConnectionLostError('The connection was unexpectedly lost') + raise ConnectionLostError('The connection was unexpectedly lost') from exc def _send_connection_closed_poison_pill(self): frame = frames.ConnectionClosedPoisonPillFrame() From e410050c4505e77b9967ef07598f2c59715c98a2 Mon Sep 17 00:00:00 2001 From: Mike Lenzen Date: Sun, 24 May 2015 08:40:36 +0600 Subject: [PATCH 017/118] Stop squashing OsError into ConnectionRefusedError when connecting --- examples/reconnecting.py | 10 ++++++---- src/asynqp/__init__.py | 8 +------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/examples/reconnecting.py b/examples/reconnecting.py index a8b257a..0f4640a 100644 --- a/examples/reconnecting.py +++ b/examples/reconnecting.py @@ -84,7 +84,8 @@ def setup_producer(connection): def start(loop): ''' Creates a connection, starts the consumer and producer. - If it fails to reconnect, it waits 1 second then try again + If it fails, it will attempt to reconnect after waiting + 1 second ''' global CONNECTION global CONSUMER @@ -93,7 +94,8 @@ def start(loop): CONNECTION = yield from setup_connection(loop) CONSUMER = yield from setup_consumer(CONNECTION) PRODUCER = loop.create_task(setup_producer(CONNECTION)) - except ConnectionError: + # Multiple exceptions may be thrown, ConnectionError, OsError + except Exception: print('failed to connect, trying again.') yield from asyncio.sleep(1) loop.create_task(start(loop)) @@ -121,7 +123,7 @@ def stop(): try: yield from CONNECTION.close() except InvalidStateError: - pass # should be automatically closed, so this is expected + pass # could be automatically closed, so this is expected CONNECTION = None @@ -131,7 +133,7 @@ def connection_lost_handler(loop, context): ConnectionErrors. The exceptions we can catch follow this inheritance scheme - | + - ConnectionError - base | - asynqp.exceptions.ConnectionClosedError - connection closed properly diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index f4f02de..c687ac1 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -48,13 +48,7 @@ def connect(host='localhost', kwargs['port'] = port dispatcher = Dispatcher() - try: - transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) - except (ConnectionRefusedError, OSError) as e: - # Throw a single exception instead of two - raise ConnectionRefusedError('Failed to connect - host: {} port: {}' - ' username: {} password: {} virtual_host: {}' - .format(host, port, username, password, virtual_host)) from e + transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) connection = yield from open_connection(loop, transport, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) return connection From 7dbfa335e2e001cb0dfd2d091e9608e573e01b15 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 2 Jun 2015 18:52:47 +0100 Subject: [PATCH 018/118] Fix #22 by killing futures when ChannelClose arrives --- src/asynqp/bases.py | 2 +- src/asynqp/channel.py | 1 + src/asynqp/routing.py | 21 ++++++++++----------- test/channel_tests.py | 9 +++++++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py index ca89e22..f00e1c4 100644 --- a/src/asynqp/bases.py +++ b/src/asynqp/bases.py @@ -21,4 +21,4 @@ def handle(self, frame): meth(frame) def handle_ConnectionClosedPoisonPillFrame(self, frame): - self.synchroniser.notify_connection_closed() + self.synchroniser.killall(ConnectionError) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 500426d..7cf16e2 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -230,6 +230,7 @@ def handle_ContentBodyFrame(self, frame): def handle_ChannelClose(self, frame): self.sender.send_CloseOK() + self.synchroniser.killall(ConnectionError) def handle_ChannelCloseOK(self, frame): self.synchroniser.notify(spec.ChannelCloseOK) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 4427d27..6af4432 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -59,9 +59,9 @@ def notify(self, method, result=None): fut.set_result(result) self._futures.remove_item(fut) - def notify_connection_closed(self): + def killall(self, exc): self.connection_closed = True - # Give a proper notification to blocking methods + # Give a proper notification to methods which are waiting for closure for method in self._blocking_methods: while True: try: @@ -72,13 +72,9 @@ def notify_connection_closed(self): # Set an exception for all others for method in self._futures.keys(): if method not in self._blocking_methods: - while True: - try: - fut = self._futures.get_leftmost(method) - fut.set_exception(ConnectionError) - self._futures.remove_item(fut) - except StopIteration: - break + for fut in self._futures.get_all(method): + fut.set_exception(exc) + self._futures.remove_item(fut) def create_reader_and_writer(handler): @@ -135,8 +131,11 @@ def remove_item(self, item): def get_leftmost(self, key): return self._items[key].first() + def get_all(self, key): + return list(self._items[key]) + def keys(self): - return self._items.keys() + return (k for k, v in self._items.items() if v) class OrderedSet(collections.MutableSet): @@ -165,4 +164,4 @@ def discard(self, item): pass def first(self): - return next(iter(self._map)) + return next(iter(self)) diff --git a/test/channel_tests.py b/test/channel_tests.py index 81ea08c..6c454f3 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -97,14 +97,19 @@ def it_should_not_throw_an_exception(self): class WhenAnUnexpectedChannelCloseArrives(OpenChannelContext): def given_we_are_awaiting_QueueDeclareOK(self): - self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.task = asyncio.async(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.tick() def when_ChannelClose_arrives(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(403, "i just dont like you", 50, 10)) + self.tick() def it_should_send_ChannelCloseOK(self): self.server.should_have_received_method(self.channel.id, spec.ChannelCloseOK()) + def it_should_throw_an_exception(self): + assert self.task.exception() is not None + class WhenSettingQOS(OpenChannelContext): def when_we_are_setting_prefetch_count_only(self): From c2be32babf2de7d1a434a892e35f85d7c354b8a1 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 4 Jun 2015 00:58:29 +0200 Subject: [PATCH 019/118] Signed/unsigned serialization and expanded table serialization Explicitly use unsigned serialization for unsigned integers Expand table serialization for more types (integers, bool, dict) --- src/asynqp/amqptypes.py | 77 ++++++++++++++++++++++++++------ src/asynqp/message.py | 16 +++---- src/asynqp/serialisation.py | 88 +++++++++++++++++++++++++++++++++---- test/serialisation_tests.py | 17 +++++++ 4 files changed, 168 insertions(+), 30 deletions(-) diff --git a/src/asynqp/amqptypes.py b/src/asynqp/amqptypes.py index dcead71..231585a 100644 --- a/src/asynqp/amqptypes.py +++ b/src/asynqp/amqptypes.py @@ -1,13 +1,6 @@ import datetime from . import serialisation - -MAX_OCTET = 0xFF -MAX_SHORT = 0xFFFF -MAX_LONG = 0xFFFFFFFF -MAX_LONG_LONG = 0xFFFFFFFFFFFFFFFF - - class Bit(object): def __init__(self, value): if isinstance(value, type(self)): @@ -33,8 +26,10 @@ def read(cls, stream): class Octet(int): + MIN = 0 + MAX = (1<<8)-1 def __new__(cls, value): - if not (0 <= value <= MAX_OCTET): + if not (Octet.MIN <= value <= Octet.MAX): raise TypeError('Could not construct an Octet from value {}'.format(value)) return super().__new__(cls, value) @@ -47,8 +42,10 @@ def read(cls, stream): class Short(int): + MIN = -(1<<15) + MAX = (1<<15)-1 def __new__(cls, value): - if not (0 <= value <= MAX_SHORT): + if not (Short.MIN <= value <= Short.MAX): raise TypeError('Could not construct a Short from value {}'.format(value)) return super().__new__(cls, value) @@ -60,9 +57,26 @@ def read(cls, stream): return cls(serialisation.read_short(stream)) +class UnsignedShort(int): + MIN = 0 + MAX = (1<<16)-1 + def __new__(cls, value): + if not (UnsignedShort.MIN <= value <= UnsignedShort.MAX): + raise TypeError('Could not construct an UnsignedShort from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_short(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_short(stream)) + class Long(int): + MIN = -(1<<31) + MAX = (1<<31)-1 def __new__(cls, value): - if not (0 <= value <= MAX_LONG): + if not (Long.MIN <= value <= Long.MAX): raise TypeError('Could not construct a Long from value {}'.format(value)) return super().__new__(cls, value) @@ -73,10 +87,28 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_long(stream)) +class UnsignedLong(int): + MIN = 0 + MAX = (1<<32)-1 + + def __new__(cls, value): + if not (UnsignedLong.MIN <= value <= UnsignedLong.MAX): + raise TypeError('Could not construct a UnsignedLong from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_long(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_long(stream)) class LongLong(int): + MIN = -(1<<63) + MAX = (1<<63)-1 + def __new__(cls, value): - if not (0 <= value <= MAX_LONG_LONG): + if not (LongLong.MIN <= value <= LongLong.MAX): raise TypeError('Could not construct a LongLong from value {}'.format(value)) return super().__new__(cls, value) @@ -87,10 +119,26 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_long_long(stream)) +class UnsignedLongLong(int): + MIN = 0 + MAX = (1<<64)-1 + + def __new__(cls, value): + if not (UnsignedLongLong.MIN <= value <= UnsignedLongLong.MAX): + raise TypeError('Could not construct a UnsignedLongLong from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_long_long(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_long_long(stream)) + class ShortStr(str): def __new__(cls, value): - if len(value) > MAX_OCTET: + if len(value) > Octet.MAX: raise TypeError('Could not construct a ShortStr from value {}'.format(value)) return super().__new__(cls, value) @@ -107,7 +155,7 @@ def read(cls, stream): class LongStr(str): def __new__(cls, value): - if len(value) > MAX_LONG: + if len(value) > UnsignedLong.MAX: raise TypeError('Could not construct a LongStr from value {}'.format(value)) return super().__new__(cls, value) @@ -154,8 +202,11 @@ def read(cls, stream): 'bit': Bit, 'octet': Octet, 'short': Short, + 'unsignedshort': UnsignedShort, 'long': Long, + 'unsignedlong': UnsignedLong, 'longlong': LongLong, + 'unsignedlonglong': UnsignedLongLong, 'table': Table, 'longstr': LongStr, 'shortstr': ShortStr, diff --git a/src/asynqp/message.py b/src/asynqp/message.py index 1881bca..ff1c4ad 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -180,9 +180,9 @@ def __eq__(self, other): and self.properties == other.properties) def write(self, stream): - stream.write(serialisation.pack_short(self.class_id)) - stream.write(serialisation.pack_short(0)) # weight - stream.write(serialisation.pack_long_long(self.body_length)) + stream.write(serialisation.pack_unsigned_short(self.class_id)) + stream.write(serialisation.pack_unsigned_short(0)) # weight + stream.write(serialisation.pack_unsigned_long_long(self.body_length)) bytesio = BytesIO() @@ -195,17 +195,17 @@ def write(self, stream): val.write(bytesio) bitshift -= 1 - stream.write(serialisation.pack_short(property_flags)) + stream.write(serialisation.pack_unsigned_short(property_flags)) stream.write(bytesio.getvalue()) @classmethod def read(cls, raw): bytesio = BytesIO(raw) - class_id = serialisation.read_short(bytesio) - weight = serialisation.read_short(bytesio) + class_id = serialisation.read_unsigned_short(bytesio) + weight = serialisation.read_unsigned_short(bytesio) assert weight == 0 - body_length = serialisation.read_long_long(bytesio) - property_flags_short = serialisation.read_short(bytesio) + body_length = serialisation.read_unsigned_long_long(bytesio) + property_flags_short = serialisation.read_unsigned_short(bytesio) properties = [] for amqptype, flag in zip(Message.property_types.values(), bin(property_flags_short)[2:]): diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index d3add45..d38ba56 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -27,16 +27,26 @@ def read_octet(stream): def read_short(stream): return _read_short(stream)[0] +@rethrow_as(struct.error, AMQPError('failed to read an unsigned short')) +def read_unsigned_short(stream): + return _read_unsigned_short(stream)[0] @rethrow_as(struct.error, AMQPError('failed to read a long')) def read_long(stream): return _read_long(stream)[0] +@rethrow_as(struct.error, AMQPError('failed to read an unsigned long')) +def read_unsigned_long(stream): + return _read_unsigned_long(stream)[0] -@rethrow_as(struct.error, AMQPError('failed to read a long')) +@rethrow_as(struct.error, AMQPError('failed to read a long long')) def read_long_long(stream): return _read_long_long(stream)[0] +@rethrow_as(struct.error, AMQPError('failed to read an unsigned long long')) +def read_unsigned_long_long(stream): + return _read_unsigned_long_long(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read a short string')) def read_short_string(stream): @@ -72,7 +82,13 @@ def _read_table(stream): b't': _read_bool, b's': _read_short_string, b'S': _read_long_string, - b'F': _read_table + b'F': _read_table, + b'u': _read_unsigned_short, + b'U': _read_short, + b'i': _read_unsigned_long, + b'I': _read_long, + b'l': _read_unsigned_long_long, + b'L': _read_long_long, } consumed = 0 @@ -103,7 +119,7 @@ def _read_short_string(stream): def _read_long_string(stream): - str_length, x = _read_long(stream) + str_length, x = _read_unsigned_long(stream) bytestring = stream.read(str_length) if len(bytestring) != str_length: raise AMQPError("Long string had incorrect length") @@ -121,16 +137,29 @@ def _read_bool(stream): def _read_short(stream): + x, = struct.unpack('!h', stream.read(2)) + return x, 2 + +def _read_unsigned_short(stream): x, = struct.unpack('!H', stream.read(2)) return x, 2 def _read_long(stream): + x, = struct.unpack('!l', stream.read(4)) + return x, 4 + +def _read_unsigned_long(stream): x, = struct.unpack('!L', stream.read(4)) return x, 4 def _read_long_long(stream): + x, = struct.unpack('!q', stream.read(8)) + return x, 8 + + +def _read_unsigned_long_long(stream): x, = struct.unpack('!Q', stream.read(8)) return x, 8 @@ -152,11 +181,42 @@ def pack_long_string(string): def pack_table(d): bytes = b'' for key, value in d.items(): - if not isinstance(value, str): - raise NotImplementedError() bytes += pack_short_string(key) - bytes += b'S' # todo: more values - bytes += pack_long_string(value) + # todo: more values + if isinstance(value, dict): + bytes += b'F' + bytes += pack_table(value) + elif isinstance(value, str): + bytes += b'S' + bytes += pack_long_string(value) + elif isinstance(value, int): + if value < 0: + if value.bit_length() < 16: + bytes += b'U' + bytes += pack_short(value) + elif value.bit_length() < 32: + bytes += b'I' + bytes += pack_long(value) + else: + bytes += b'L' + bytes += pack_long_long(value) + else: + if value.bit_length() <= 16: + bytes += b'u' + bytes += pack_unsigned_short(value) + elif value.bit_length() <= 32: + bytes += b'i' + bytes += pack_unsigned_long(value) + else: + bytes += b'l' + bytes += pack_unsigned_long_long(value) + + elif isinstance(value, bool): + bytes += b't' + bytes += pack_bool(value) + else: + raise NotImplementedError() + val = pack_long(len(bytes)) + bytes return val @@ -164,19 +224,29 @@ def pack_table(d): def pack_octet(number): return struct.pack('!B', number) - def pack_short(number): - return struct.pack('!H', number) + return struct.pack('!h', number) +def pack_unsigned_short(number): + return struct.pack('!H', number) def pack_long(number): + return struct.pack('!l', number) + +def pack_unsigned_long(number): return struct.pack('!L', number) def pack_long_long(number): + return struct.pack('!q', number) + +def pack_unsigned_long_long(number): return struct.pack('!Q', number) +def pack_bool(b): + return struct.pack('!?', b) + def pack_bools(*bs): tot = 0 for n, b in enumerate(bs): diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 551afc7..55be35d 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -17,6 +17,23 @@ def because_we_read_the_table(self, bytes, expected): def it_should_return_the_table(self, bytes, expected): assert self.result == expected +class WhenPackingAndUnpackingATable: + @classmethod + def examples_of_tables(cls): + for encoded, table in WhenParsingATable.examples_of_tables(): + yield table + yield { 'a': (1<<16), 'b': (1<<15) } + yield { 'c': 65535, 'd': -65535 } + yield { 'e': -65536 } + yield { 'f': -(1<<63), 'g': ((1<<64)-1)} + yield { 'f': (1<<32), 'g': (1<<63)} + + def because_we_pack_and_unpack_the_table(self, table): + self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) + + def it_should_return_the_table(self, table): + assert self.result == table + class WhenParsingABadTable: @classmethod From 85cd89fd42da43f665c40e81b1a4dfae7db69fbf Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 4 Jun 2015 07:58:37 +0200 Subject: [PATCH 020/118] Fix formatting --- src/asynqp/amqptypes.py | 29 +++++++++++++++++++---------- src/asynqp/serialisation.py | 13 +++++++++++++ test/serialisation_tests.py | 11 ++++++----- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/asynqp/amqptypes.py b/src/asynqp/amqptypes.py index 231585a..3c95cc4 100644 --- a/src/asynqp/amqptypes.py +++ b/src/asynqp/amqptypes.py @@ -1,6 +1,7 @@ import datetime from . import serialisation + class Bit(object): def __init__(self, value): if isinstance(value, type(self)): @@ -27,7 +28,8 @@ def read(cls, stream): class Octet(int): MIN = 0 - MAX = (1<<8)-1 + MAX = (1 << 8) - 1 + def __new__(cls, value): if not (Octet.MIN <= value <= Octet.MAX): raise TypeError('Could not construct an Octet from value {}'.format(value)) @@ -42,8 +44,9 @@ def read(cls, stream): class Short(int): - MIN = -(1<<15) - MAX = (1<<15)-1 + MIN = -(1 << 15) + MAX = (1 << 15) - 1 + def __new__(cls, value): if not (Short.MIN <= value <= Short.MAX): raise TypeError('Could not construct a Short from value {}'.format(value)) @@ -59,7 +62,8 @@ def read(cls, stream): class UnsignedShort(int): MIN = 0 - MAX = (1<<16)-1 + MAX = (1 << 16) - 1 + def __new__(cls, value): if not (UnsignedShort.MIN <= value <= UnsignedShort.MAX): raise TypeError('Could not construct an UnsignedShort from value {}'.format(value)) @@ -72,9 +76,11 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_unsigned_short(stream)) + class Long(int): - MIN = -(1<<31) - MAX = (1<<31)-1 + MIN = -(1 << 31) + MAX = (1 << 31) - 1 + def __new__(cls, value): if not (Long.MIN <= value <= Long.MAX): raise TypeError('Could not construct a Long from value {}'.format(value)) @@ -87,9 +93,10 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_long(stream)) + class UnsignedLong(int): MIN = 0 - MAX = (1<<32)-1 + MAX = (1 << 32) - 1 def __new__(cls, value): if not (UnsignedLong.MIN <= value <= UnsignedLong.MAX): @@ -103,9 +110,10 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_unsigned_long(stream)) + class LongLong(int): - MIN = -(1<<63) - MAX = (1<<63)-1 + MIN = -(1 << 63) + MAX = (1 << 63) - 1 def __new__(cls, value): if not (LongLong.MIN <= value <= LongLong.MAX): @@ -119,9 +127,10 @@ def write(self, stream): def read(cls, stream): return cls(serialisation.read_long_long(stream)) + class UnsignedLongLong(int): MIN = 0 - MAX = (1<<64)-1 + MAX = (1 << 64) - 1 def __new__(cls, value): if not (UnsignedLongLong.MIN <= value <= UnsignedLongLong.MAX): diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index d38ba56..f4836a9 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -27,22 +27,27 @@ def read_octet(stream): def read_short(stream): return _read_short(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read an unsigned short')) def read_unsigned_short(stream): return _read_unsigned_short(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read a long')) def read_long(stream): return _read_long(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read an unsigned long')) def read_unsigned_long(stream): return _read_unsigned_long(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read a long long')) def read_long_long(stream): return _read_long_long(stream)[0] + @rethrow_as(struct.error, AMQPError('failed to read an unsigned long long')) def read_unsigned_long_long(stream): return _read_unsigned_long_long(stream)[0] @@ -140,6 +145,7 @@ def _read_short(stream): x, = struct.unpack('!h', stream.read(2)) return x, 2 + def _read_unsigned_short(stream): x, = struct.unpack('!H', stream.read(2)) return x, 2 @@ -149,6 +155,7 @@ def _read_long(stream): x, = struct.unpack('!l', stream.read(4)) return x, 4 + def _read_unsigned_long(stream): x, = struct.unpack('!L', stream.read(4)) return x, 4 @@ -224,15 +231,19 @@ def pack_table(d): def pack_octet(number): return struct.pack('!B', number) + def pack_short(number): return struct.pack('!h', number) + def pack_unsigned_short(number): return struct.pack('!H', number) + def pack_long(number): return struct.pack('!l', number) + def pack_unsigned_long(number): return struct.pack('!L', number) @@ -240,6 +251,7 @@ def pack_unsigned_long(number): def pack_long_long(number): return struct.pack('!q', number) + def pack_unsigned_long_long(number): return struct.pack('!Q', number) @@ -247,6 +259,7 @@ def pack_unsigned_long_long(number): def pack_bool(b): return struct.pack('!?', b) + def pack_bools(*bs): tot = 0 for n, b in enumerate(bs): diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 55be35d..00c1442 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -17,16 +17,17 @@ def because_we_read_the_table(self, bytes, expected): def it_should_return_the_table(self, bytes, expected): assert self.result == expected + class WhenPackingAndUnpackingATable: @classmethod def examples_of_tables(cls): for encoded, table in WhenParsingATable.examples_of_tables(): yield table - yield { 'a': (1<<16), 'b': (1<<15) } - yield { 'c': 65535, 'd': -65535 } - yield { 'e': -65536 } - yield { 'f': -(1<<63), 'g': ((1<<64)-1)} - yield { 'f': (1<<32), 'g': (1<<63)} + yield {'a': (1 << 16), 'b': (1 << 15)} + yield {'c': 65535, 'd': -65535} + yield {'e': -65536} + yield {'f': -(1 << 63), 'g': ((1 << 64) - 1)} + yield {'f': (1 << 32), 'g': (1 << 63)} def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) From 3dd0d57b13516f0c925932ccf550e0e4f1abfe7c Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Thu, 4 Jun 2015 13:21:07 +0100 Subject: [PATCH 021/118] Update date --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 3db08be..00694a5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2014 Benjamin Hodgson +Copyright (c) 2013-2015 Benjamin Hodgson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 130a85ffce91da078d6f6b5a337a90e54b68584a Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 12:08:52 +0200 Subject: [PATCH 022/118] Table length should be unsigned long --- src/asynqp/serialisation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index f4836a9..28ec502 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -99,7 +99,7 @@ def _read_table(stream): consumed = 0 table = {} - table_length, initial_long_size = _read_long(stream) + table_length, initial_long_size = _read_unsigned_long(stream) consumed += initial_long_size while consumed < table_length + initial_long_size: From c0ff4d9140010e1f9f7a45d1689f96cfbe5327dd Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 15:24:18 +0200 Subject: [PATCH 023/118] Implement timestamp serialisation --- src/asynqp/serialisation.py | 20 ++++++++++++++++++++ test/serialisation_tests.py | 27 ++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index 28ec502..a4c57a5 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -1,5 +1,6 @@ import struct from .exceptions import AMQPError +from datetime import datetime def rethrow_as(expected_cls, to_throw): @@ -81,6 +82,11 @@ def read_bools(byte, number_of_bools): return (b == "1" for b in reversed(bits)) +@rethrow_as(struct.error, AMQPError('failed to read a boolean')) +def read_time_stamp(stream): + return _read_time_stamp(stream)[0] + + def _read_table(stream): # TODO: more value types TABLE_VALUE_PARSERS = { @@ -94,6 +100,7 @@ def _read_table(stream): b'I': _read_long, b'l': _read_unsigned_long_long, b'L': _read_long_long, + b'T': _read_time_stamp } consumed = 0 @@ -171,6 +178,11 @@ def _read_unsigned_long_long(stream): return x, 8 +def _read_time_stamp(stream): + x, = struct.unpack('!Q', stream.read(8)) + return datetime.fromtimestamp(x * 1e-3), 8 + + ########################################################### # Serialisation ########################################################### @@ -196,6 +208,9 @@ def pack_table(d): elif isinstance(value, str): bytes += b'S' bytes += pack_long_string(value) + elif isinstance(value, datetime): + bytes += b'T' + bytes += pack_time_stamp(value) elif isinstance(value, int): if value < 0: if value.bit_length() < 16: @@ -260,6 +275,11 @@ def pack_bool(b): return struct.pack('!?', b) +def pack_time_stamp(timeval): + number = int(timeval.timestamp() * 1e3) + return struct.pack('!Q', number) + + def pack_bools(*bs): tot = 0 for n, b in enumerate(bs): diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 00c1442..9c74e83 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -1,5 +1,6 @@ from io import BytesIO import contexts +import datetime from asynqp import serialisation, AMQPError @@ -28,6 +29,7 @@ def examples_of_tables(cls): yield {'e': -65536} yield {'f': -(1 << 63), 'g': ((1 << 64) - 1)} yield {'f': (1 << 32), 'g': (1 << 63)} + yield {'t': datetime.datetime(2038, 1, 1, 3, 14, 9, 123000) } def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) @@ -35,7 +37,6 @@ def because_we_pack_and_unpack_the_table(self, table): def it_should_return_the_table(self, table): assert self.result == table - class WhenParsingABadTable: @classmethod def examples_of_bad_tables(self): @@ -79,3 +80,27 @@ def because_I_pack_them(self, bools, expected): def it_should_pack_them_correctly(self, bools, expected): assert self.result == expected + +class WhenParsingATimeStamp: + @classmethod + def examples_of_time_stamps(cls): + yield b'\x00\x00\x01\xf3\xa3\x16\x9d\xe3', datetime.datetime(2038, 1, 1, 3, 14, 9, 123000) + + def because_we_read_a_time_stamp(self, binary, _): + self.result = serialisation.read_time_stamp(BytesIO(binary)) + + def it_should_reat_it_correctly(self, _, expected): + assert self.result == expected + +class WhenWritingATimeStamp: + @classmethod + def examples_of_time_stamps(cls): + for encoded, timeval in WhenParsingATimeStamp.examples_of_time_stamps(): + yield timeval, encoded + + def because_I_pack_them(self, timeval, _): + print(repr(timeval)) + self.result = serialisation.pack_time_stamp(timeval) + + def it_should_pack_them_correctly(self, _, expected): + assert self.result == expected From 397f97987dff5d0c3d58327a8afdfbcf398eeedb Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 15:38:52 +0200 Subject: [PATCH 024/118] Fixed formatting for tests --- test/serialisation_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 9c74e83..65d5bca 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -29,7 +29,7 @@ def examples_of_tables(cls): yield {'e': -65536} yield {'f': -(1 << 63), 'g': ((1 << 64) - 1)} yield {'f': (1 << 32), 'g': (1 << 63)} - yield {'t': datetime.datetime(2038, 1, 1, 3, 14, 9, 123000) } + yield {'t': datetime.datetime(2038, 1, 1, 3, 14, 9, 123000)} def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) @@ -37,6 +37,7 @@ def because_we_pack_and_unpack_the_table(self, table): def it_should_return_the_table(self, table): assert self.result == table + class WhenParsingABadTable: @classmethod def examples_of_bad_tables(self): @@ -81,6 +82,7 @@ def because_I_pack_them(self, bools, expected): def it_should_pack_them_correctly(self, bools, expected): assert self.result == expected + class WhenParsingATimeStamp: @classmethod def examples_of_time_stamps(cls): @@ -92,6 +94,7 @@ def because_we_read_a_time_stamp(self, binary, _): def it_should_reat_it_correctly(self, _, expected): assert self.result == expected + class WhenWritingATimeStamp: @classmethod def examples_of_time_stamps(cls): From f2d2d437244dd169511052cbcd60200f7cbdf734 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 23:11:09 +0200 Subject: [PATCH 025/118] Fix typo in test description --- test/serialisation_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 65d5bca..8804258 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -91,7 +91,7 @@ def examples_of_time_stamps(cls): def because_we_read_a_time_stamp(self, binary, _): self.result = serialisation.read_time_stamp(BytesIO(binary)) - def it_should_reat_it_correctly(self, _, expected): + def it_should_read_it_correctly(self, _, expected): assert self.result == expected From 44bfcf5d19dd5fabb677ff4f952c36a613cf4f0a Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sun, 7 Jun 2015 08:32:33 +0200 Subject: [PATCH 026/118] Fix internal timestamp representation to utc Previously, the timezone was not set, which added an offset depending on the timezone set on the computer (or not, if set to utc) --- src/asynqp/serialisation.py | 5 +++-- test/serialisation_tests.py | 29 +++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index a4c57a5..3c5b3b5 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -1,6 +1,6 @@ import struct from .exceptions import AMQPError -from datetime import datetime +from datetime import datetime, timezone def rethrow_as(expected_cls, to_throw): @@ -180,7 +180,8 @@ def _read_unsigned_long_long(stream): def _read_time_stamp(stream): x, = struct.unpack('!Q', stream.read(8)) - return datetime.fromtimestamp(x * 1e-3), 8 + # From datetime.fromutctimestamp converts it to a local timestamp without timezone information + return datetime.fromtimestamp(x * 1e-3, timezone.utc), 8 ########################################################### diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 8804258..eb2b1a6 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -1,6 +1,6 @@ from io import BytesIO import contexts -import datetime +from datetime import datetime, timezone, timedelta from asynqp import serialisation, AMQPError @@ -29,7 +29,6 @@ def examples_of_tables(cls): yield {'e': -65536} yield {'f': -(1 << 63), 'g': ((1 << 64) - 1)} yield {'f': (1 << 32), 'g': (1 << 63)} - yield {'t': datetime.datetime(2038, 1, 1, 3, 14, 9, 123000)} def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) @@ -86,7 +85,14 @@ def it_should_pack_them_correctly(self, bools, expected): class WhenParsingATimeStamp: @classmethod def examples_of_time_stamps(cls): - yield b'\x00\x00\x01\xf3\xa3\x16\x9d\xe3', datetime.datetime(2038, 1, 1, 3, 14, 9, 123000) + # The timestamp should be zero relative to epoch + yield b'\x00\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, tzinfo=timezone.utc) + # And independent of the timezone + yield b'\x00\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, 1, 30, tzinfo=timezone(timedelta(hours=1, minutes=30))) + # And and increase by a millisecond + yield b'\x00\x00\x00\x00\x00\x00\x00\x01', datetime(1970, 1, 1, microsecond=1000, tzinfo=timezone.utc) + # Cannot validate, that it is unsigned, as it is + # yield b'\x80\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, microsecond=1000, tzinfo=timezone.utc) def because_we_read_a_time_stamp(self, binary, _): self.result = serialisation.read_time_stamp(BytesIO(binary)) @@ -102,8 +108,23 @@ def examples_of_time_stamps(cls): yield timeval, encoded def because_I_pack_them(self, timeval, _): - print(repr(timeval)) self.result = serialisation.pack_time_stamp(timeval) def it_should_pack_them_correctly(self, _, expected): assert self.result == expected + + +class WhenPackingAndUnpackingATimeStamp: + # Ensure, we do not add some offset by the serialisation process + @classmethod + def examples_of_time_stamps(cls): + yield datetime(1970, 1, 1, tzinfo=timezone.utc) + yield datetime(1979, 1, 1, tzinfo=timezone(timedelta(hours=1, minutes=30))) + + def because_I_pack_them(self, timeval): + packed = serialisation.pack_time_stamp(timeval) + unpacked = serialisation.read_time_stamp(BytesIO(packed)) + self.result = unpacked - timeval + + def it_should_pack_them_correctly(self, timeval): + assert abs(self.result.total_seconds()) < 1.0e-9 From e7303a870cb58c32160d4147e8c4d0eafb651cc1 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sun, 7 Jun 2015 08:35:50 +0200 Subject: [PATCH 027/118] Rename time_stamp to timestamp, following the style in datetime --- src/asynqp/serialisation.py | 12 ++++++------ test/serialisation_tests.py | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index 3c5b3b5..cf3f8f5 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -83,8 +83,8 @@ def read_bools(byte, number_of_bools): @rethrow_as(struct.error, AMQPError('failed to read a boolean')) -def read_time_stamp(stream): - return _read_time_stamp(stream)[0] +def read_timestamp(stream): + return _read_timestamp(stream)[0] def _read_table(stream): @@ -100,7 +100,7 @@ def _read_table(stream): b'I': _read_long, b'l': _read_unsigned_long_long, b'L': _read_long_long, - b'T': _read_time_stamp + b'T': _read_timestamp } consumed = 0 @@ -178,7 +178,7 @@ def _read_unsigned_long_long(stream): return x, 8 -def _read_time_stamp(stream): +def _read_timestamp(stream): x, = struct.unpack('!Q', stream.read(8)) # From datetime.fromutctimestamp converts it to a local timestamp without timezone information return datetime.fromtimestamp(x * 1e-3, timezone.utc), 8 @@ -211,7 +211,7 @@ def pack_table(d): bytes += pack_long_string(value) elif isinstance(value, datetime): bytes += b'T' - bytes += pack_time_stamp(value) + bytes += pack_timestamp(value) elif isinstance(value, int): if value < 0: if value.bit_length() < 16: @@ -276,7 +276,7 @@ def pack_bool(b): return struct.pack('!?', b) -def pack_time_stamp(timeval): +def pack_timestamp(timeval): number = int(timeval.timestamp() * 1e3) return struct.pack('!Q', number) diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index eb2b1a6..247a31d 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -82,9 +82,9 @@ def it_should_pack_them_correctly(self, bools, expected): assert self.result == expected -class WhenParsingATimeStamp: +class WhenParsingATimestamp: @classmethod - def examples_of_time_stamps(cls): + def examples_of_timestamps(cls): # The timestamp should be zero relative to epoch yield b'\x00\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, tzinfo=timezone.utc) # And independent of the timezone @@ -94,36 +94,36 @@ def examples_of_time_stamps(cls): # Cannot validate, that it is unsigned, as it is # yield b'\x80\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, microsecond=1000, tzinfo=timezone.utc) - def because_we_read_a_time_stamp(self, binary, _): - self.result = serialisation.read_time_stamp(BytesIO(binary)) + def because_we_read_a_timestamp(self, binary, _): + self.result = serialisation.read_timestamp(BytesIO(binary)) def it_should_read_it_correctly(self, _, expected): assert self.result == expected -class WhenWritingATimeStamp: +class WhenWritingATimestamp: @classmethod - def examples_of_time_stamps(cls): - for encoded, timeval in WhenParsingATimeStamp.examples_of_time_stamps(): + def examples_of_timestamps(cls): + for encoded, timeval in WhenParsingATimestamp.examples_of_timestamps(): yield timeval, encoded def because_I_pack_them(self, timeval, _): - self.result = serialisation.pack_time_stamp(timeval) + self.result = serialisation.pack_timestamp(timeval) def it_should_pack_them_correctly(self, _, expected): assert self.result == expected -class WhenPackingAndUnpackingATimeStamp: +class WhenPackingAndUnpackingATimestamp: # Ensure, we do not add some offset by the serialisation process @classmethod - def examples_of_time_stamps(cls): + def examples_of_timestamps(cls): yield datetime(1970, 1, 1, tzinfo=timezone.utc) yield datetime(1979, 1, 1, tzinfo=timezone(timedelta(hours=1, minutes=30))) def because_I_pack_them(self, timeval): - packed = serialisation.pack_time_stamp(timeval) - unpacked = serialisation.read_time_stamp(BytesIO(packed)) + packed = serialisation.pack_timestamp(timeval) + unpacked = serialisation.read_timestamp(BytesIO(packed)) self.result = unpacked - timeval def it_should_pack_them_correctly(self, timeval): From 735be64959095e047d04a148c04138b61a431ff3 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 12:08:52 +0200 Subject: [PATCH 028/118] Table length should be unsigned long --- src/asynqp/serialisation.py | 4 ++-- test/serialisation_tests.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index f4836a9..216336e 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -99,7 +99,7 @@ def _read_table(stream): consumed = 0 table = {} - table_length, initial_long_size = _read_long(stream) + table_length, initial_long_size = _read_unsigned_long(stream) consumed += initial_long_size while consumed < table_length + initial_long_size: @@ -224,7 +224,7 @@ def pack_table(d): else: raise NotImplementedError() - val = pack_long(len(bytes)) + bytes + val = pack_unsigned_long(len(bytes)) + bytes return val diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 00c1442..d1591b7 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -36,6 +36,20 @@ def it_should_return_the_table(self, table): assert self.result == table +class WhenParsingAHugeTable: + @classmethod + def examples_of_huge_tables(self): + # That would be -1 for an signed int + yield b"\xFF\xFF\xFF\xFF\xFF" + + def because_we_read_the_table(self, bytes): + # We expect the serialisation to read over the bounds, but only if it is unsigned + self.exception = contexts.catch(serialisation.read_table, BytesIO(bytes)) + + def it_should_throw_an_AMQPError(self): + assert isinstance(self.exception, AMQPError) + + class WhenParsingABadTable: @classmethod def examples_of_bad_tables(self): From 73123eddc38cad13300fd1f9747f7bd80ef0eccb Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 19:33:17 +0200 Subject: [PATCH 029/118] Changed serialisation to Qpid/RabbitMq codes, added more types Apparently, the standard defined incompatible type-codes the pre-existing Qpid/Rabbit extensions, and at least rabbitmq kept them instead. Documentation: https://www.rabbitmq.com/amqp-0-9-1-errata.html#section_3 Code: https://github.com/rabbitmq/rabbitmq-server/blob/rabbitmq_v3_5_3/src/rabbit_binary_parser.erl#L41 --- src/asynqp/amqptypes.py | 2 +- src/asynqp/serialisation.py | 179 ++++++++++++++++++++++++------------ test/serialisation_tests.py | 38 +++++++- 3 files changed, 155 insertions(+), 64 deletions(-) diff --git a/src/asynqp/amqptypes.py b/src/asynqp/amqptypes.py index 3c95cc4..662f9ac 100644 --- a/src/asynqp/amqptypes.py +++ b/src/asynqp/amqptypes.py @@ -196,7 +196,7 @@ def __new__(cls, *args, **kwargs): raise TypeError("Could not construct a timestamp from value {}".format(value)) def __eq__(self, other): - return abs(self - other) < datetime.timedelta(seconds=1) + return abs(self - other) < datetime.timedelta(milliseconds=1) def write(self, stream): stamp = int(self.timestamp()) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index d762b91..c887024 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -70,6 +70,12 @@ def read_table(stream): return _read_table(stream)[0] +@rethrow_as(KeyError, AMQPError('failed to read an array')) +@rethrow_as(struct.error, AMQPError('failed to read an array')) +def read_array(stream): + return _read_array(stream)[0] + + @rethrow_as(struct.error, AMQPError('failed to read a boolean')) def read_bool(stream): return _read_bool(stream)[0] @@ -87,27 +93,31 @@ def read_timestamp(stream): return _read_timestamp(stream)[0] -def _read_table(stream): - # TODO: more value types +def qpid_rabbit_mq_table(): + # TODO: fix amqp 0.9.1 compatibility + # TODO: Add missing types TABLE_VALUE_PARSERS = { b't': _read_bool, - b's': _read_short_string, + b'b': _read_signed_byte, + b's': _read_short, + b'I': _read_long, + b'l': _read_long_long, b'S': _read_long_string, + b'A': _read_array, + b'V': _read_void, + b'x': _read_byte_array, b'F': _read_table, - b'u': _read_unsigned_short, - b'U': _read_short, - b'i': _read_unsigned_long, - b'I': _read_long, - b'l': _read_unsigned_long_long, - b'L': _read_long_long, b'T': _read_timestamp } + return TABLE_VALUE_PARSERS - consumed = 0 + +def _read_table(stream): + TABLE_VALUE_PARSERS = qpid_rabbit_mq_table() table = {} table_length, initial_long_size = _read_unsigned_long(stream) - consumed += initial_long_size + consumed = initial_long_size while consumed < table_length + initial_long_size: key, x = _read_short_string(stream) @@ -132,10 +142,10 @@ def _read_short_string(stream): def _read_long_string(stream): str_length, x = _read_unsigned_long(stream) - bytestring = stream.read(str_length) - if len(bytestring) != str_length: + buffer = stream.read(str_length) + if len(buffer) != str_length: raise AMQPError("Long string had incorrect length") - return bytestring.decode('utf-8'), x + str_length + return buffer.decode('utf-8'), x + str_length def _read_octet(stream): @@ -143,6 +153,11 @@ def _read_octet(stream): return x, 1 +def _read_signed_byte(stream): + x, = struct.unpack_from('!b', stream.read(1)) + return x, 1 + + def _read_bool(stream): x, = struct.unpack('!?', stream.read(1)) return x, 1 @@ -184,70 +199,104 @@ def _read_timestamp(stream): return datetime.fromtimestamp(x * 1e-3, timezone.utc), 8 +def _read_array(stream): + TABLE_VALUE_PARSERS = qpid_rabbit_mq_table() + field_array = [] + + # The standard says only long, but unsigned long seems sensible + array_length, initial_long_size = _read_unsigned_long(stream) + consumed = initial_long_size + + while consumed < array_length + initial_long_size: + value_type_code = stream.read(1) + consumed += 1 + value, x = TABLE_VALUE_PARSERS[value_type_code](stream) + consumed += x + field_array.append(value) + + return field_array, consumed + + +def _read_void(stream): + return None, 0 + + +def _read_byte_array(stream): + byte_array_length, x = _read_unsigned_long(stream) + return stream.read(byte_array_length), byte_array_length + x + + ########################################################### # Serialisation ########################################################### def pack_short_string(string): - bytes = string.encode('utf-8') - return pack_octet(len(bytes)) + bytes + buffer = string.encode('utf-8') + return pack_octet(len(buffer)) + buffer def pack_long_string(string): - bytes = string.encode('utf-8') - return pack_long(len(bytes)) + bytes + buffer = string.encode('utf-8') + return pack_unsigned_long(len(buffer)) + buffer + + +def pack_field_value(value): + buffer = b'' + if isinstance(value, bool): + buffer += b't' + buffer += pack_bool(value) + elif isinstance(value, dict): + buffer += b'F' + buffer += pack_table(value) + elif isinstance(value, list): + buffer += b'A' + buffer += pack_array(value) + elif isinstance(value, bytes): + buffer += b'x' + buffer += pack_byte_array(value) + elif isinstance(value, str): + buffer += b'S' + buffer += pack_long_string(value) + elif isinstance(value, datetime): + buffer += b'T' + buffer += pack_timestamp(value) + elif isinstance(value, int): + if value.bit_length() < 8: + buffer += b'b' + buffer += pack_signed_byte(value) + elif value.bit_length() < 32: + buffer += b'I' + buffer += pack_long(value) + else: + raise NotImplementedError() + else: + raise NotImplementedError() + + return buffer def pack_table(d): - bytes = b'' + buffer = b'' for key, value in d.items(): - bytes += pack_short_string(key) + buffer += pack_short_string(key) # todo: more values - if isinstance(value, dict): - bytes += b'F' - bytes += pack_table(value) - elif isinstance(value, str): - bytes += b'S' - bytes += pack_long_string(value) - elif isinstance(value, datetime): - bytes += b'T' - bytes += pack_timestamp(value) - elif isinstance(value, int): - if value < 0: - if value.bit_length() < 16: - bytes += b'U' - bytes += pack_short(value) - elif value.bit_length() < 32: - bytes += b'I' - bytes += pack_long(value) - else: - bytes += b'L' - bytes += pack_long_long(value) - else: - if value.bit_length() <= 16: - bytes += b'u' - bytes += pack_unsigned_short(value) - elif value.bit_length() <= 32: - bytes += b'i' - bytes += pack_unsigned_long(value) - else: - bytes += b'l' - bytes += pack_unsigned_long_long(value) - - elif isinstance(value, bool): - bytes += b't' - bytes += pack_bool(value) - else: - raise NotImplementedError() + buffer += pack_field_value(value) - val = pack_unsigned_long(len(bytes)) + bytes - return val + return pack_unsigned_long(len(buffer)) + buffer def pack_octet(number): return struct.pack('!B', number) +def pack_signed_byte(number): + return struct.pack('!b', number) + + +def pack_unsigned_byte(number): + return struct.pack('!B', number) + + def pack_short(number): return struct.pack('!h', number) @@ -281,6 +330,20 @@ def pack_timestamp(timeval): return struct.pack('!Q', number) +def pack_byte_array(value): + buffer = pack_unsigned_long(len(value)) + buffer += value + return buffer + + +def pack_array(items): + buffer = b'' + for value in items: + buffer += pack_field_value(value) + + return pack_unsigned_long(len(buffer)) + buffer + + def pack_bools(*bs): tot = 0 for n, b in enumerate(bs): diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index b48d528..b75685d 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -7,8 +7,15 @@ class WhenParsingATable: @classmethod def examples_of_tables(self): + yield b"\x00\x00\x00\x00", {} yield b"\x00\x00\x00\x0E\x04key1t\x00\x04key2t\x01", {'key1': False, 'key2': True} - yield b"\x00\x00\x00\x0B\x03keys\x05hello", {'key': 'hello'} + yield b"\x00\x00\x00\x06\x03keyb\xff", {'key': -1} + yield b"\x00\x00\x00\x07\x03keys\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x09\x03keyI\xff\xff\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x0C\x03keyl\xff\xff\xff\xff\xff\xff\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x05\x03keyV", {'key': None} + yield b"\x00\x00\x00\x05\x03keyA\x00\x00\x00\x00", {'key': []} + yield b"\x00\x00\x00\x0C\x03keyx\x00\x00\x00\x04\x00\x01\x02\x03", {'key': b"\x00\x01\x02\x03"} yield b"\x00\x00\x00\x0E\x03keyS\x00\x00\x00\x05hello", {'key': 'hello'} yield b"\x00\x00\x00\x16\x03keyF\x00\x00\x00\x0D\x0Aanotherkeyt\x00", {'key': {'anotherkey': False}} @@ -22,13 +29,12 @@ def it_should_return_the_table(self, bytes, expected): class WhenPackingAndUnpackingATable: @classmethod def examples_of_tables(cls): - for encoded, table in WhenParsingATable.examples_of_tables(): - yield table yield {'a': (1 << 16), 'b': (1 << 15)} yield {'c': 65535, 'd': -65535} yield {'e': -65536} - yield {'f': -(1 << 63), 'g': ((1 << 64) - 1)} - yield {'f': (1 << 32), 'g': (1 << 63)} + yield {'f': -0x7FFFFFFF, 'g': 0x7FFFFFFF} + yield {'x': b"\x01\x02"} + yield {'x': []} def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) @@ -64,6 +70,28 @@ def it_should_throw_an_AMQPError(self): assert isinstance(self.exception, AMQPError) +class WhenParsingAnArray: + @classmethod + def examples_of_arrays(self): + yield b"\x00\x00\x00\x00", [] + yield b"\x00\x00\x00\x04t\x00t\x01", [False, True] + yield b"\x00\x00\x00\x02b\xff", [-1] + yield b"\x00\x00\x00\x03s\xff\xff", [-1] + yield b"\x00\x00\x00\x05I\xff\xff\xff\xff", [-1] + yield b"\x00\x00\x00\x09l\xff\xff\xff\xff\xff\xff\xff\xff", [-1] + yield b"\x00\x00\x00\x01V", [None] + yield b"\x00\x00\x00\x05A\x00\x00\x00\x00", [[]] + yield b"\x00\x00\x00\x09x\x00\x00\x00\x04\x00\x01\x02\x03", [b"\x00\x01\x02\x03"] + yield b"\x00\x00\x00\x0AS\x00\x00\x00\x05hello", ['hello'] + yield b"\x00\x00\x00\x12F\x00\x00\x00\x0D\x0Aanotherkeyt\x00", [{'anotherkey': False}] + + def because_we_read_the_array(self, buffer, expected): + self.result = serialisation.read_array(BytesIO(buffer)) + + def it_should_return_the_array(self, buffer, expected): + assert self.result == expected + + class WhenParsingALongString: def because_we_read_a_long_string(self): self.result = serialisation.read_long_string(BytesIO(b"\x00\x00\x00\x05hello")) From 611d2fe8a8c4e84e08225ef6afd09a76bbc738ae Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 6 Jun 2015 19:33:17 +0200 Subject: [PATCH 030/118] Changed serialisation to Qpid/RabbitMq codes, added more types Apparently, the standard defined incompatible type-codes the pre-existing Qpid/Rabbit extensions, and at least rabbitmq kept them instead. Documentation: https://www.rabbitmq.com/amqp-0-9-1-errata.html#section_3 Code: https://github.com/rabbitmq/rabbitmq-server/blob/rabbitmq_v3_5_3/src/rabbit_binary_parser.erl#L41 --- src/asynqp/serialisation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index c887024..9aa7fa3 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -92,7 +92,6 @@ def read_bools(byte, number_of_bools): def read_timestamp(stream): return _read_timestamp(stream)[0] - def qpid_rabbit_mq_table(): # TODO: fix amqp 0.9.1 compatibility # TODO: Add missing types From 75326461f5d6b31b9330bd74335b5046b78b575f Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 10 Jun 2015 17:01:21 +0200 Subject: [PATCH 031/118] Pass table arguments to broker This enables the use of extensions, like the message- or queue-ttl (http://www.rabbitmq.com/extensions.html) The attributes should be ignored, if the broker does not understand them --- src/asynqp/channel.py | 28 ++++++++++++++-------------- src/asynqp/queue.py | 16 ++++++++-------- src/asynqp/serialisation.py | 1 + test/integration_tests.py | 16 +++++++++++++++- test/queue_tests.py | 20 ++++++++++---------- 5 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 7cf16e2..625b637 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -37,7 +37,7 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factor self.reader = reader @asyncio.coroutine - def declare_exchange(self, name, type, *, durable=True, auto_delete=False, internal=False): + def declare_exchange(self, name, type, *, durable=True, auto_delete=False, internal=False, arguments=None): """ Declare an :class:`Exchange` on the broker. If the exchange does not exist, it will be created. @@ -59,14 +59,14 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " "and do not begin with 'amq.'") - self.sender.send_ExchangeDeclare(name, type, durable, auto_delete, internal) + self.sender.send_ExchangeDeclare(name, type, durable, auto_delete, internal, arguments or {}) yield from self.synchroniser.await(spec.ExchangeDeclareOK) ex = exchange.Exchange(self.reader, self.synchroniser, self.sender, name, type, durable, auto_delete, internal) self.reader.ready() return ex @asyncio.coroutine - def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=False): + def declare_queue(self, name=None, *, durable=True, exclusive=False, auto_delete=False, arguments=None): """ Declare a queue on the broker. If the queue does not exist, it will be created. @@ -85,7 +85,7 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=F :return: The new :class:`Queue` object. """ - q = yield from self.queue_factory.declare(name, durable, exclusive, auto_delete) + q = yield from self.queue_factory.declare(name or '', durable, exclusive, auto_delete, arguments or {}) return q @asyncio.coroutine @@ -331,20 +331,20 @@ def __init__(self, channel_id, protocol, connection_info): def send_ChannelOpen(self): self.send_method(spec.ChannelOpen('')) - def send_ExchangeDeclare(self, name, type, durable, auto_delete, internal): - self.send_method(spec.ExchangeDeclare(0, name, type, False, durable, auto_delete, internal, False, {})) + def send_ExchangeDeclare(self, name, type, durable, auto_delete, internal, arguments): + self.send_method(spec.ExchangeDeclare(0, name, type, False, durable, auto_delete, internal, False, arguments)) def send_ExchangeDelete(self, name, if_unused): self.send_method(spec.ExchangeDelete(0, name, if_unused, False)) - def send_QueueDeclare(self, name, durable, exclusive, auto_delete): - self.send_method(spec.QueueDeclare(0, name, False, durable, exclusive, auto_delete, False, {})) + def send_QueueDeclare(self, name, durable, exclusive, auto_delete, arguments): + self.send_method(spec.QueueDeclare(0, name, False, durable, exclusive, auto_delete, False, arguments)) - def send_QueueBind(self, queue_name, exchange_name, routing_key): - self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, {})) + def send_QueueBind(self, queue_name, exchange_name, routing_key, arguments): + self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, arguments)) - def send_QueueUnbind(self, queue_name, exchange_name, routing_key): - self.send_method(spec.QueueUnbind(0, queue_name, exchange_name, routing_key, {})) + def send_QueueUnbind(self, queue_name, exchange_name, routing_key, arguments): + self.send_method(spec.QueueUnbind(0, queue_name, exchange_name, routing_key, arguments)) def send_QueuePurge(self, queue_name): self.send_method(spec.QueuePurge(0, queue_name, False)) @@ -356,8 +356,8 @@ def send_BasicPublish(self, exchange_name, routing_key, mandatory, message): self.send_method(spec.BasicPublish(0, exchange_name, routing_key, mandatory, False)) self.send_content(message) - def send_BasicConsume(self, queue_name, no_local, no_ack, exclusive): - self.send_method(spec.BasicConsume(0, queue_name, '', no_local, no_ack, exclusive, False, {})) + def send_BasicConsume(self, queue_name, no_local, no_ack, exclusive, arguments): + self.send_method(spec.BasicConsume(0, queue_name, '', no_local, no_ack, exclusive, False, arguments)) def send_BasicCancel(self, consumer_tag): self.send_method(spec.BasicCancel(consumer_tag, False)) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index fa76bc3..d6aae4e 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -46,7 +46,7 @@ def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclu self.deleted = False @asyncio.coroutine - def bind(self, exchange, routing_key): + def bind(self, exchange, routing_key, arguments=None): """ Bind a queue to an exchange, with the supplied routing key. @@ -63,14 +63,14 @@ def bind(self, exchange, routing_key): if self.deleted: raise Deleted("Queue {} was deleted".format(self.name)) - self.sender.send_QueueBind(self.name, exchange.name, routing_key) + self.sender.send_QueueBind(self.name, exchange.name, routing_key, arguments or {}) yield from self.synchroniser.await(spec.QueueBindOK) b = QueueBinding(self.reader, self.sender, self.synchroniser, self, exchange, routing_key) self.reader.ready() return b @asyncio.coroutine - def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False): + def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, arguments=None): """ Start a consumer on the queue. Messages will be delivered asynchronously to the consumer. The callback function will be called whenever a new message arrives on the queue. @@ -89,7 +89,7 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False): if self.deleted: raise Deleted("Queue {} was deleted".format(self.name)) - self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive) + self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive, arguments or {}) tag = yield from self.synchroniser.await(spec.BasicConsumeOK) consumer = Consumer(tag, callback, self.sender, self.synchroniser, self.reader) self.consumers.add_consumer(consumer) @@ -188,7 +188,7 @@ def __init__(self, reader, sender, synchroniser, queue, exchange, routing_key): self.deleted = False @asyncio.coroutine - def unbind(self): + def unbind(self, arguments=None): """ Unbind the queue from the exchange. @@ -197,7 +197,7 @@ def unbind(self): if self.deleted: raise Deleted("Queue {} was already unbound from exchange {}".format(self.queue.name, self.exchange.name)) - self.sender.send_QueueUnbind(self.queue.name, self.exchange.name, self.routing_key) + self.sender.send_QueueUnbind(self.queue.name, self.exchange.name, self.routing_key, arguments or {}) yield from self.synchroniser.await(spec.QueueUnbindOK) self.deleted = True self.reader.ready() @@ -254,13 +254,13 @@ def __init__(self, sender, synchroniser, reader, consumers): self.consumers = consumers @asyncio.coroutine - def declare(self, name, durable, exclusive, auto_delete): + def declare(self, name, durable, exclusive, auto_delete, arguments=None): if not VALID_QUEUE_NAME_RE.match(name): raise ValueError("Not a valid queue name.\n" "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " "and do not begin with 'amq.'") - self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete) + self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments or {}) name = yield from self.synchroniser.await(spec.QueueDeclareOK) q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete) self.reader.ready() diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index 9aa7fa3..c887024 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -92,6 +92,7 @@ def read_bools(byte, number_of_bools): def read_timestamp(stream): return _read_timestamp(stream)[0] + def qpid_rabbit_mq_table(): # TODO: fix amqp 0.9.1 compatibility # TODO: Add missing types diff --git a/test/integration_tests.py b/test/integration_tests.py index 15153bf..ef7c29e 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -1,7 +1,10 @@ import asyncio import asynqp import socket +import urllib.request +import base64 import contexts +import json from .util import testing_exception_handler @@ -84,12 +87,23 @@ def cleanup_the_channel(self): class WhenDeclaringAQueue(ChannelContext): + ARGUMENTS = {'x-expires': 300, 'x-message-ttl': 1000, 'x-table-test': {'a': [1, 'a', {}, []], 'c': 1}} + def when_I_declare_a_queue(self): - self.queue = self.loop.run_until_complete(asyncio.wait_for(self.channel.declare_queue('my.queue', exclusive=True), 0.2)) + self.queue = self.loop.run_until_complete(asyncio.wait_for(self.channel.declare_queue('my.queue', exclusive=True, arguments=WhenDeclaringAQueue.ARGUMENTS), 0.2)) def it_should_have_the_correct_queue_name(self): assert self.queue.name == 'my.queue' + def it_should_have_the_correct_attributes_in_rabbitmq(self): + request = urllib.request.Request("http://localhost:15672/api/queues/%2f/{}".format(self.queue.name)) + base64string = base64.encodebytes(b'guest:guest').rstrip() + request.add_header("Authorization", b"Basic " + base64string) + result = urllib.request.urlopen(request) + result = json.loads(result.read().decode('utf-8')) + arguments = result['arguments'] + assert arguments == WhenDeclaringAQueue.ARGUMENTS + def cleanup_the_queue(self): self.loop.run_until_complete(asyncio.wait_for(self.queue.delete(if_unused=False, if_empty=False), 0.2)) diff --git a/test/queue_tests.py b/test/queue_tests.py index dd3ad45..6f8ba32 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -10,17 +10,17 @@ class WhenDeclaringAQueue(OpenChannelContext): def when_I_declare_a_queue(self): - self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) def it_should_send_a_QueueDeclare_method(self): - expected_method = spec.QueueDeclare(0, 'my.nice.queue', False, True, True, True, False, {}) + expected_method = spec.QueueDeclare(0, 'my.nice.queue', False, True, True, True, False, {'x-expires': 300, 'x-message-ttl': 1000}) self.server.should_have_received_method(self.channel.id, expected_method) class WhenQueueDeclareOKArrives(OpenChannelContext): def given_I_declared_a_queue(self): self.queue_name = 'my.nice.queue' - self.task = asyncio.async(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True)) + self.task = asyncio.async(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) self.tick() def when_QueueDeclareOK_arrives(self): @@ -72,10 +72,10 @@ def it_should_throw_ValueError(self): class WhenBindingAQueueToAnExchange(QueueContext, ExchangeContext): def when_I_bind_the_queue(self): - self.async_partial(self.queue.bind(self.exchange, 'routing.key')) + self.async_partial(self.queue.bind(self.exchange, 'routing.key', arguments={'x-ignore': ''})) def it_should_send_QueueBind(self): - expected_method = spec.QueueBind(0, self.queue.name, self.exchange.name, 'routing.key', False, {}) + expected_method = spec.QueueBind(0, self.queue.name, self.exchange.name, 'routing.key', False, {'x-ignore': ''}) self.server.should_have_received_method(self.channel.id, expected_method) @@ -97,10 +97,10 @@ def and_the_returned_binding_should_have_the_correct_exchange(self): class WhenUnbindingAQueue(BoundQueueContext): def when_I_unbind_the_queue(self): - self.async_partial(self.binding.unbind()) + self.async_partial(self.binding.unbind(arguments={'x-ignore': ''})) def it_should_send_QueueUnbind(self): - expected_method = spec.QueueUnbind(0, self.queue.name, self.exchange.name, 'routing.key', {}) + expected_method = spec.QueueUnbind(0, self.queue.name, self.exchange.name, 'routing.key', {'x-ignore': ''}) self.server.should_have_received_method(self.channel.id, expected_method) @@ -179,15 +179,15 @@ def it_should_put_the_routing_key_on_the_msg(self): class WhenISubscribeToAQueue(QueueContext): def when_I_start_a_consumer(self): - self.async_partial(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False)) + self.async_partial(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) def it_should_send_BasicConsume(self): - self.server.should_have_received_method(self.channel.id, spec.BasicConsume(0, self.queue.name, '', False, False, False, False, {})) + self.server.should_have_received_method(self.channel.id, spec.BasicConsume(0, self.queue.name, '', False, False, False, False, {'x-priority': 1})) class WhenConsumeOKArrives(QueueContext): def given_I_started_a_consumer(self): - self.task = asyncio.async(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False)) + self.task = asyncio.async(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() def when_BasicConsumeOK_arrives(self): From 3762023f5b968b8d9da1d09283f6cc639f5e94a3 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 14 Jun 2015 14:58:28 +0100 Subject: [PATCH 032/118] trial import of setuptools --- setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 44b586f..873eeed 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ -from ez_setup import use_setuptools -use_setuptools() +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() - -from setuptools import setup, find_packages + from setuptools import setup, find_packages setup( From c31da9cea9382f4ace08124eda2ffcde490a84c1 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 14 Jun 2015 15:31:18 +0100 Subject: [PATCH 033/118] Documentation; minor API fixes --- CHANGELOG.md | 18 ++++++++++++++++++ doc/extensions.rst | 11 +++++++++++ doc/index.rst | 6 +----- src/asynqp/__init__.py | 13 ++++++++++--- src/asynqp/channel.py | 14 ++++++++------ src/asynqp/queue.py | 8 +++++--- 6 files changed, 53 insertions(+), 17 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 doc/extensions.rst diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..98d4ccc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +What's new in `asynqp` +====================== + +v0.4 +---- + +* Improved error handling. + When the connection to the server is lost, + any futures awaiting communication from the server will now be cancelled. + (Thanks to @lenzenmi, in pull request #19) +* Support for custom RabbitMQ extensions by an `arguments` keyword parameter for a number of methods. + (Thanks to @fweisel, in pull request #27) +* Improved compatibility with RabbitMQ's implementation of the + wire protocol, including better support for tables. + (Thanks to @fweisel, in pull requests #24, #25, #26, #28) +* Support for a `sock` keyword argument to `asynqp.connect`. + (Thanks to @urbaniak, in pull request #14) + diff --git a/doc/extensions.rst b/doc/extensions.rst new file mode 100644 index 0000000..55b93fd --- /dev/null +++ b/doc/extensions.rst @@ -0,0 +1,11 @@ +.. _extensions: + +Protocol extensions +=================== + +RabbitMQ, and other brokers, support certain extensions to the AMQP protocol. +`asynqp`'s support for such extensions currently includes +*optional extra arguments* to certain methods such as :meth:`Channel.declare_queue() `. + +The acceptable parameters for optional argument dictionaries is implementation-dependent. +See`RabbitMQ's supported extensions `. diff --git a/doc/index.rst b/doc/index.rst index 1f91134..aff3286 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,8 +1,3 @@ -.. asynqp documentation master file, created by - sphinx-quickstart on Wed Jun 11 20:44:10 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - ``asynqp`` ========== An AMQP (aka `RabbitMQ `_) client library for :mod:`asyncio`. @@ -68,6 +63,7 @@ Table of contents reference conformance + extensions * :ref:`genindex` * :ref:`modindex` diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index c687ac1..807b02c 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -20,7 +20,7 @@ def connect(host='localhost', port=5672, username='guest', password='guest', virtual_host='/', *, - loop=None, **kwargs): + loop=None, sock=None, **kwargs): """ Connect to an AMQP server on the given host and port. @@ -32,8 +32,13 @@ def connect(host='localhost', :param str username: the username to authenticate with. :param str password: the password to authenticate with. :param str virtual_host: the AMQP virtual host to connect to. + :keyword BaseEventLoop loop: An instance of :class:`~asyncio.BaseEventLoop` to use. + (Defaults to :func:`asyncio.get_event_loop()`) + :keyword socket sock: A :func:`~socket.socket` instance to use for the connection. + This is passed on to :meth:`loop.create_connection() `. + If ``sock`` is supplied then ``host`` and ``port`` will be ignored. - Further keyword arguments are passed on to :meth:`create_connection() `. + Further keyword arguments are passed on to :meth:`loop.create_connection() `. :return: the :class:`Connection` object. """ @@ -43,9 +48,11 @@ def connect(host='localhost', loop = asyncio.get_event_loop() if loop is None else loop - if 'sock' not in kwargs: + if sock is None: kwargs['host'] = host kwargs['port'] = port + else: + kwargs['sock'] = sock dispatcher = Dispatcher() transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 625b637..749a642 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -44,10 +44,14 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter This method is a :ref:`coroutine `. :param str name: the name of the exchange. - :param str type: the type of the exchange (usually one of ``'fanout'``, ``'direct'``, ``'topic'``, or ``'headers'``) + :param str type: the type of the exchange + (usually one of ``'fanout'``, ``'direct'``, ``'topic'``, or ``'headers'``) :keyword bool durable: If true, the exchange will be re-created when the server restarts. - :keyword bool auto_delete: If true, the exchange will be deleted when the last queue is un-bound from it. - :keyword bool internal: If true, the exchange cannot be published to directly; it can only be bound to other exchanges. + :keyword bool auto_delete: If true, the exchange will be + deleted when the last queue is un-bound from it. + :keyword bool internal: If true, the exchange cannot be published to directly; + it can only be bound to other exchanges. + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: the new :class:`Exchange` object. """ @@ -74,14 +78,12 @@ def declare_queue(self, name=None, *, durable=True, exclusive=False, auto_delete :param str name: the name of the queue. Supplying a name of '' will create a queue with a unique name of the server's choosing. - :keyword bool durable: If true, the queue will be re-created when the server restarts. - :keyword bool exclusive: If true, the queue can only be accessed by the current connection, and will be deleted when the connection is closed. - :keyword bool auto_delete: If true, the queue will be deleted when the last consumer is cancelled. If there were never any conusmers, the queue won't be deleted. + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`Queue` object. """ diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index d6aae4e..ce6b19f 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -46,7 +46,7 @@ def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclu self.deleted = False @asyncio.coroutine - def bind(self, exchange, routing_key, arguments=None): + def bind(self, exchange, routing_key, *, arguments=None): """ Bind a queue to an exchange, with the supplied routing key. @@ -57,6 +57,7 @@ def bind(self, exchange, routing_key, arguments=None): :param asynqp.Exchange exchange: the :class:`Exchange` to bind to :param str routing_key: the routing key under which to bind + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`QueueBinding` object """ @@ -83,6 +84,7 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, ar published by this connection. :keyword bool no_ack: If true, messages delivered to the consumer don't require acknowledgement. :keyword bool exclusive: If true, only this consumer can access the queue. + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The newly created :class:`Consumer` object. """ @@ -254,13 +256,13 @@ def __init__(self, sender, synchroniser, reader, consumers): self.consumers = consumers @asyncio.coroutine - def declare(self, name, durable, exclusive, auto_delete, arguments=None): + def declare(self, name, durable, exclusive, auto_delete, arguments): if not VALID_QUEUE_NAME_RE.match(name): raise ValueError("Not a valid queue name.\n" "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " "and do not begin with 'amq.'") - self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments or {}) + self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments) name = yield from self.synchroniser.await(spec.QueueDeclareOK) q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete) self.reader.ready() From aa2070809dafb308721e70d4a337f95452952ced Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 14 Jun 2015 15:46:30 +0100 Subject: [PATCH 034/118] Add examples to docs --- README.md | 9 +++-- doc/examples.rst | 14 ++++++++ doc/examples/helloworld.py | 40 +++++++++++++++++++++ {examples => doc/examples}/reconnecting.py | 15 +++----- doc/index.rst | 42 ++-------------------- 5 files changed, 68 insertions(+), 52 deletions(-) create mode 100644 doc/examples.rst create mode 100644 doc/examples/helloworld.py rename {examples => doc/examples}/reconnecting.py (93%) diff --git a/README.md b/README.md index 5e0aa3b..01fcc01 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,10 @@ import asynqp @asyncio.coroutine -def send_and_receive(): +def hello_world(): + """ + Sends a 'hello world' message and then reads it from the queue. + """ # connect to the RabbitMQ broker connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') @@ -34,7 +37,7 @@ def send_and_receive(): yield from queue.bind(exchange, 'routing.key') # If you pass in a dict it will be automatically converted to JSON - msg = asynqp.Message({'test_body': 'content'}) + msg = asynqp.Message({'hello': 'world'}) exchange.publish(msg, 'routing.key') # Synchronously get a message from the queue @@ -50,7 +53,7 @@ def send_and_receive(): if __name__ == "__main__": loop = asyncio.get_event_loop() - loop.run_until_complete(send_and_receive()) + loop.run_until_complete(hello_world()) ``` diff --git a/doc/examples.rst b/doc/examples.rst new file mode 100644 index 0000000..d5d8d2e --- /dev/null +++ b/doc/examples.rst @@ -0,0 +1,14 @@ +Examples +======== + + +Hello World +----------- +.. literalinclude:: /examples/helloworld.py + :language: python + + +Reconnecting +------------ +.. literalinclude:: /examples/reconnecting.py + :language: python diff --git a/doc/examples/helloworld.py b/doc/examples/helloworld.py new file mode 100644 index 0000000..896e9b7 --- /dev/null +++ b/doc/examples/helloworld.py @@ -0,0 +1,40 @@ +import asyncio +import asynqp + + +@asyncio.coroutine +def hello_world(): + """ + Sends a 'hello world' message and then reads it from the queue. + """ + # connect to the RabbitMQ broker + connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') + + # Open a communications channel + channel = yield from connection.open_channel() + + # Create a queue and an exchange on the broker + exchange = yield from channel.declare_exchange('test.exchange', 'direct') + queue = yield from channel.declare_queue('test.queue') + + # Bind the queue to the exchange, so the queue will get messages published to the exchange + yield from queue.bind(exchange, 'routing.key') + + # If you pass in a dict it will be automatically converted to JSON + msg = asynqp.Message({'hello': 'world'}) + exchange.publish(msg, 'routing.key') + + # Synchronously get a message from the queue + received_message = yield from queue.get() + print(received_message.json()) # get JSON from incoming messages easily + + # Acknowledge a delivered message + received_message.ack() + + yield from channel.close() + yield from connection.close() + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(hello_world()) diff --git a/examples/reconnecting.py b/doc/examples/reconnecting.py similarity index 93% rename from examples/reconnecting.py rename to doc/examples/reconnecting.py index 0f4640a..78831b0 100644 --- a/examples/reconnecting.py +++ b/doc/examples/reconnecting.py @@ -2,12 +2,8 @@ Example async consumer and publisher that will reconnect automatically when a connection to rabbitmq is broken and restored. - - -.. note:: - - No attempt is made to re-send messages that are generated - while the connection is down. +Note that no attempt is made to re-send messages that are +generated while the connection is down. ''' import asyncio import asynqp @@ -28,12 +24,11 @@ def setup_connection(loop): 5672, username='guest', password='guest') - return connection @asyncio.coroutine -def setup_queue_and_exchange(connection): +def setup_exchange_and_queue(connection): # Open a communications channel channel = yield from connection.open_channel() @@ -57,7 +52,7 @@ def callback(msg): print('Received: {}'.format(msg.body)) msg.ack() - _, queue = yield from setup_queue_and_exchange(connection) + _, queue = yield from setup_exchange_and_queue(connection) # connect the callback to the queue consumer = yield from queue.consume(callback) @@ -70,7 +65,7 @@ def setup_producer(connection): The producer will live as an asyncio.Task to stop it call Task.cancel() ''' - exchange, _ = yield from setup_queue_and_exchange(connection) + exchange, _ = yield from setup_exchange_and_queue(connection) count = 0 while True: diff --git a/doc/index.rst b/doc/index.rst index aff3286..e3e402c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -5,45 +5,8 @@ An AMQP (aka `RabbitMQ `_) client library for :mod:`as Example ------- -:: - - import asyncio - import asynqp - - - @asyncio.coroutine - def send_and_receive(): - # connect to the RabbitMQ broker - connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') - - # Open a communications channel - channel = yield from connection.open_channel() - - # Create a queue and an exchange on the broker - exchange = yield from channel.declare_exchange('test.exchange', 'direct') - queue = yield from channel.declare_queue('test.queue') - - # Bind the queue to the exchange, so the queue will get messages published to the exchange - yield from queue.bind(exchange, 'routing.key') - - # If you pass in a dict it will be automatically converted to JSON - msg = asynqp.Message({'test_body': 'content'}) - exchange.publish(msg, 'routing.key') - - # Synchronously get a message from the queue - received_message = yield from queue.get() - print(received_message.json()) # get JSON from incoming messages easily - - # Acknowledge a delivered message - received_message.ack() - - yield from channel.close() - yield from connection.close() - - - if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(send_and_receive()) +.. literalinclude:: /examples/helloworld.py + :language: python Installation @@ -62,6 +25,7 @@ Table of contents :maxdepth: 2 reference + examples conformance extensions From 93d37033af8ee17a93b2bdc28cd2b65c54499658 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 30 Jun 2015 13:40:14 +0100 Subject: [PATCH 035/118] Fixed a test on my windows machine. It was refusing to call the rabbit API. Rather than debug that, I just relaxed the test a little. It's already covered by unit tests, so the integration test now basically just checks for errors --- src/asynqp/channel.py | 4 ++-- src/asynqp/queue.py | 9 +++++++-- test/integration_tests.py | 14 +++----------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 749a642..a0c2ffa 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -70,7 +70,7 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter return ex @asyncio.coroutine - def declare_queue(self, name=None, *, durable=True, exclusive=False, auto_delete=False, arguments=None): + def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=False, arguments=None): """ Declare a queue on the broker. If the queue does not exist, it will be created. @@ -87,7 +87,7 @@ def declare_queue(self, name=None, *, durable=True, exclusive=False, auto_delete :return: The new :class:`Queue` object. """ - q = yield from self.queue_factory.declare(name or '', durable, exclusive, auto_delete, arguments or {}) + q = yield from self.queue_factory.declare(name, durable, exclusive, auto_delete, arguments if arguments is not None else {}) return q @asyncio.coroutine diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index ce6b19f..d094c7d 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -32,8 +32,12 @@ class Queue(object): .. attribute:: auto_delete if True, the queue will be deleted when its last consumer is removed + + .. attribute:: arguments + + A dictionary of the extra arguments that were used to declare the queue. """ - def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete): + def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments): self.reader = reader self.consumers = consumers self.synchroniser = synchroniser @@ -43,6 +47,7 @@ def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclu self.durable = durable self.exclusive = exclusive self.auto_delete = auto_delete + self.arguments = arguments self.deleted = False @asyncio.coroutine @@ -264,7 +269,7 @@ def declare(self, name, durable, exclusive, auto_delete, arguments): self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments) name = yield from self.synchroniser.await(spec.QueueDeclareOK) - q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete) + q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete, arguments) self.reader.ready() return q diff --git a/test/integration_tests.py b/test/integration_tests.py index ef7c29e..8bdfa55 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -1,10 +1,7 @@ import asyncio import asynqp import socket -import urllib.request -import base64 import contexts -import json from .util import testing_exception_handler @@ -90,19 +87,14 @@ class WhenDeclaringAQueue(ChannelContext): ARGUMENTS = {'x-expires': 300, 'x-message-ttl': 1000, 'x-table-test': {'a': [1, 'a', {}, []], 'c': 1}} def when_I_declare_a_queue(self): - self.queue = self.loop.run_until_complete(asyncio.wait_for(self.channel.declare_queue('my.queue', exclusive=True, arguments=WhenDeclaringAQueue.ARGUMENTS), 0.2)) + coro = self.channel.declare_queue('my.queue', exclusive=True, arguments=WhenDeclaringAQueue.ARGUMENTS) + self.queue = self.loop.run_until_complete(asyncio.wait_for(coro, 0.2)) def it_should_have_the_correct_queue_name(self): assert self.queue.name == 'my.queue' def it_should_have_the_correct_attributes_in_rabbitmq(self): - request = urllib.request.Request("http://localhost:15672/api/queues/%2f/{}".format(self.queue.name)) - base64string = base64.encodebytes(b'guest:guest').rstrip() - request.add_header("Authorization", b"Basic " + base64string) - result = urllib.request.urlopen(request) - result = json.loads(result.read().decode('utf-8')) - arguments = result['arguments'] - assert arguments == WhenDeclaringAQueue.ARGUMENTS + assert self.queue.arguments == WhenDeclaringAQueue.ARGUMENTS def cleanup_the_queue(self): self.loop.run_until_complete(asyncio.wait_for(self.queue.delete(if_unused=False, if_empty=False), 0.2)) From 08b69307200b773270d939e8535169218b0c29b4 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 30 Jun 2015 16:08:51 +0100 Subject: [PATCH 036/118] #23 throw more specific exceptions --- src/asynqp/_exceptions.py | 2 ++ src/asynqp/bases.py | 4 ++-- src/asynqp/channel.py | 8 +++++--- src/asynqp/connection.py | 6 +++--- src/asynqp/exceptions.py | 18 ++++++++++++++---- src/asynqp/frames.py | 2 +- src/asynqp/protocol.py | 2 +- src/asynqp/spec.py | 14 ++++++++++++++ test/channel_tests.py | 10 +++++----- 9 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 src/asynqp/_exceptions.py diff --git a/src/asynqp/_exceptions.py b/src/asynqp/_exceptions.py new file mode 100644 index 0000000..dcf784c --- /dev/null +++ b/src/asynqp/_exceptions.py @@ -0,0 +1,2 @@ +class AMQPError(IOError): + pass diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py index f00e1c4..2ea7c5e 100644 --- a/src/asynqp/bases.py +++ b/src/asynqp/bases.py @@ -7,7 +7,7 @@ def send_method(self, method): self.protocol.send_method(self.channel_id, method) -class FrameHandler(object): +class Actor(object): def __init__(self, synchroniser, sender): self.synchroniser = synchroniser self.sender = sender @@ -20,5 +20,5 @@ def handle(self, frame): meth(frame) - def handle_ConnectionClosedPoisonPillFrame(self, frame): + def handle_PoisonPillFrame(self, frame): self.synchroniser.killall(ConnectionError) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index a0c2ffa..5a3f80e 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -4,6 +4,7 @@ from . import frames from . import spec from . import queue +from . import exceptions from . import exchange from . import message from . import routing @@ -163,7 +164,7 @@ def open(self): consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - handler = ChannelFrameHandler(synchroniser, sender) + handler = ChannelActor(synchroniser, sender) reader, writer = routing.create_reader_and_writer(handler) handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) @@ -184,7 +185,7 @@ def open(self): return channel -class ChannelFrameHandler(bases.FrameHandler): +class ChannelActor(bases.Actor): def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) @@ -232,7 +233,8 @@ def handle_ContentBodyFrame(self, frame): def handle_ChannelClose(self, frame): self.sender.send_CloseOK() - self.synchroniser.killall(ConnectionError) + exc = exceptions.get_exception_type(frame.payload.reply_code) + self.synchroniser.killall(exc) def handle_ChannelCloseOK(self, frame): self.synchroniser.notify(spec.ChannelCloseOK) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 7d952ee..226e0d2 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -49,7 +49,7 @@ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, # this is ugly. when the connection is closing, all methods other than ConnectionCloseOK # should be ignored. at the moment this behaviour is part of the dispatcher - # but this introduces an extra dependency between Connection and ConnectionFrameHandler which + # but this introduces an extra dependency between Connection and ConnectionActor which # i don't like self.closing = asyncio.Future() self.closing.add_done_callback(lambda fut: dispatcher.closing.set_result(fut.result())) @@ -84,7 +84,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): sender = ConnectionMethodSender(protocol) connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) - handler = ConnectionFrameHandler(synchroniser, sender, protocol, connection) + handler = ConnectionActor(synchroniser, sender, protocol, connection) reader, writer = routing.create_reader_and_writer(handler) @@ -122,7 +122,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): return connection -class ConnectionFrameHandler(bases.FrameHandler): +class ConnectionActor(bases.Actor): def __init__(self, synchroniser, sender, protocol, connection): super().__init__(synchroniser, sender) self.protocol = protocol diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 0372751..755b6c2 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -1,3 +1,7 @@ +from ._exceptions import AMQPError +from .spec import EXCEPTIONS, CONSTANTS_INVERSE + + __all__ = [ "AMQPError", "ConnectionClosedError", @@ -5,10 +9,7 @@ "UndeliverableMessage", "Deleted" ] - - -class AMQPError(IOError): - pass +__all__.extend(EXCEPTIONS.keys()) class ConnectionClosedError(ConnectionError): @@ -32,3 +33,12 @@ class UndeliverableMessage(ValueError): class Deleted(ValueError): pass + + +globals().update(EXCEPTIONS) + + +def get_exception_type(reply_code): + name = CONSTANTS_INVERSE[reply_code] + classname = ''.join([x.capitalize() for x in name.split('_')]) + return EXCEPTIONS[classname] diff --git a/src/asynqp/frames.py b/src/asynqp/frames.py index 72e3d47..412e1d0 100644 --- a/src/asynqp/frames.py +++ b/src/asynqp/frames.py @@ -66,7 +66,7 @@ def __init__(self): pass -class ConnectionClosedPoisonPillFrame(Frame): +class PoisonPillFrame(Frame): channel_id = 0 payload = b'' diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 6ae1c35..af4e3fd 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -55,7 +55,7 @@ def connection_lost(self, exc): raise ConnectionLostError('The connection was unexpectedly lost') from exc def _send_connection_closed_poison_pill(self): - frame = frames.ConnectionClosedPoisonPillFrame() + frame = frames.PoisonPillFrame() # send the poison pill to all open queues self.dispatcher.dispatch_all(frame) diff --git a/src/asynqp/spec.py b/src/asynqp/spec.py index 2c7d9ac..72a7aad 100644 --- a/src/asynqp/spec.py +++ b/src/asynqp/spec.py @@ -6,6 +6,7 @@ from . import amqptypes from . import serialisation from .amqptypes import FIELD_TYPES +from ._exceptions import AMQPError def read_method(raw): @@ -164,7 +165,20 @@ def generate_methods(classes): return methods +def generate_exceptions(constants): + ret = {} + + for name, value in constants.items(): + if 300 <= value < 600: # it's an error + classname = ''.join([x.capitalize() for x in name.split('_')]) + ret[classname] = type(classname, (AMQPError,), {}) + + return ret + + METHODS, CONSTANTS = load_spec() +CONSTANTS_INVERSE = {value: name for name, value in CONSTANTS.items()} +EXCEPTIONS = generate_exceptions(CONSTANTS) # what the hack? 'response' is always a table but the protocol spec says it's a longstr. METHODS['ConnectionStartOK'].field_info['response'] = amqptypes.Table diff --git a/test/channel_tests.py b/test/channel_tests.py index 6c454f3..1af4024 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -2,7 +2,7 @@ import contexts import asynqp from unittest import mock -from asynqp import spec, frames +from asynqp import spec, frames, exceptions from asynqp import message from . import util from .base_contexts import OpenConnectionContext, OpenChannelContext @@ -49,7 +49,7 @@ def it_should_send_ChannelClose(self): class WhenTheServerClosesAChannel(OpenChannelContext): def when_the_server_shuts_the_channel_down(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(404, 'i am tired of you', 40, 50)) def it_should_send_ChannelCloseOK(self): self.server.should_have_received_method(self.channel.id, spec.ChannelCloseOK()) @@ -69,7 +69,7 @@ def it_MUST_discard_the_method(self): class WhenAnotherMethodArrivesAfterTheServerClosedTheChannel(OpenChannelContext): def given_the_server_closed_the_channel(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(404, 'i am tired of you', 40, 50)) self.server.reset() def when_another_method_arrives(self): @@ -101,14 +101,14 @@ def given_we_are_awaiting_QueueDeclareOK(self): self.tick() def when_ChannelClose_arrives(self): - self.server.send_method(self.channel.id, spec.ChannelClose(403, "i just dont like you", 50, 10)) + self.server.send_method(self.channel.id, spec.ChannelClose(406, "the precondition, she failed", 50, 10)) self.tick() def it_should_send_ChannelCloseOK(self): self.server.should_have_received_method(self.channel.id, spec.ChannelCloseOK()) def it_should_throw_an_exception(self): - assert self.task.exception() is not None + assert isinstance(self.task.exception(), exceptions.PreconditionFailed) class WhenSettingQOS(OpenChannelContext): From 7c5c06f0cb5463f57400438cde8d1217475dc10b Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 30 Jun 2015 16:27:54 +0100 Subject: [PATCH 037/118] add exception docs and made function private --- doc/reference.rst | 8 ++++++++ src/asynqp/channel.py | 2 +- src/asynqp/exceptions.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 57583bb..2acef7e 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -66,3 +66,11 @@ Message objects .. autoclass:: IncomingMessage :members: + + +Exceptions +---------- + +.. automodule:: asynqp.exceptions + :members: + :undoc-members: diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 5a3f80e..9bc1919 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -233,7 +233,7 @@ def handle_ContentBodyFrame(self, frame): def handle_ChannelClose(self, frame): self.sender.send_CloseOK() - exc = exceptions.get_exception_type(frame.payload.reply_code) + exc = exceptions._get_exception_type(frame.payload.reply_code) self.synchroniser.killall(exc) def handle_ChannelCloseOK(self, frame): diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 755b6c2..cc1b378 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -38,7 +38,7 @@ class Deleted(ValueError): globals().update(EXCEPTIONS) -def get_exception_type(reply_code): +def _get_exception_type(reply_code): name = CONSTANTS_INVERSE[reply_code] classname = ''.join([x.capitalize() for x in name.split('_')]) return EXCEPTIONS[classname] From 1536ac09e784f553b4b6adc3e07ad303adafb2d7 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 30 Jun 2015 16:32:34 +0100 Subject: [PATCH 038/118] fix __all__ --- src/asynqp/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index 807b02c..b7f86f9 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,5 +1,5 @@ import asyncio -from .exceptions import AMQPError, UndeliverableMessage, Deleted +from .exceptions import * # noqa from .message import Message, IncomingMessage from .connection import Connection from .channel import Channel @@ -8,11 +8,11 @@ __all__ = [ - "AMQPError", "UndeliverableMessage", "Deleted", "Message", "IncomingMessage", "Connection", "Channel", "Exchange", "Queue", "QueueBinding", "Consumer", "connect", "connect_and_open_channel" ] +__all__ += exceptions.__all__ @asyncio.coroutine From 3e6ce9934760e1742fd421b9025324faa9ac94ce Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 30 Jun 2015 16:42:38 +0100 Subject: [PATCH 039/118] Increment version to 0.4 --- CHANGELOG.md | 7 ++++--- TODO | 3 +-- doc/conf.py | 6 +++--- setup.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98d4ccc..91a39b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,10 @@ v0.4 ---- * Improved error handling. - When the connection to the server is lost, - any futures awaiting communication from the server will now be cancelled. - (Thanks to @lenzenmi, in pull request #19) + * When the connection to the server is lost, any futures awaiting communication + from the server will now be cancelled. + (Thanks to @lenzenmi, in pull request #19) + * More detailed exceptions on channel closure. * Support for custom RabbitMQ extensions by an `arguments` keyword parameter for a number of methods. (Thanks to @fweisel, in pull request #27) * Improved compatibility with RabbitMQ's implementation of the diff --git a/TODO b/TODO index 3fdf635..f9b5f50 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ API design issues Public implementation-related fields in public classes (even though they're not documented) - Threading issues + Threading issues? Load testing? @@ -41,6 +41,5 @@ Unimplemented methods: Unimplemented functions Customise connection-tune response - General exception handling Passive declares No-wait flags diff --git a/doc/conf.py b/doc/conf.py index 396667a..a32388b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,16 +51,16 @@ # General information about the project. project = 'asynqp' -copyright = '2014, Benjamin Hodgson' +copyright = '2014-2015, Benjamin Hodgson' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.3' +version = '0.4' # The full version, including alpha/beta/rc tags. -release = '0.3' +release = '0.4' def hide_class_constructor(app, what, name, obj, options, signature, return_annotation): diff --git a/setup.py b/setup.py index 873eeed..c82d52c 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='asynqp', - version='0.3', + version='0.4', author="Benjamin Hodgson", author_email="benjamin.hodgson@huddle.net", url="https://github.com/benjamin-hodgson/asynqp", From 9254ae6f7d440f5783385792f7efd4b2a412c648 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 8 Jul 2015 16:46:24 +0100 Subject: [PATCH 040/118] #31 remove ConnectionClosedError --- src/asynqp/exceptions.py | 11 +---------- src/asynqp/protocol.py | 15 +++------------ src/asynqp/routing.py | 1 + test/integration_tests.py | 33 +++++++++++++++------------------ test/protocol_tests.py | 25 +------------------------ test/util.py | 4 ++-- 6 files changed, 23 insertions(+), 66 deletions(-) diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index cc1b378..b493309 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -4,7 +4,6 @@ __all__ = [ "AMQPError", - "ConnectionClosedError", "ConnectionLostError", "UndeliverableMessage", "Deleted" @@ -12,15 +11,7 @@ __all__.extend(EXCEPTIONS.keys()) -class ConnectionClosedError(ConnectionError): - ''' - Connection was closed normally by either the amqp server - or the client. - ''' - pass - - -class ConnectionLostError(ConnectionClosedError): +class ConnectionLostError(ConnectionError): ''' Connection was closed unexpectedly ''' diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index af4e3fd..40833e2 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -2,7 +2,7 @@ import struct from . import spec from . import frames -from .exceptions import AMQPError, ConnectionLostError, ConnectionClosedError +from .exceptions import AMQPError, ConnectionLostError class AMQP(asyncio.Protocol): @@ -30,8 +30,6 @@ def data_received(self, data): frame, remainder = result self.dispatcher.dispatch(frame) - if not remainder: - return data = remainder def send_method(self, channel, method): @@ -48,17 +46,10 @@ def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) def connection_lost(self, exc): - self._send_connection_closed_poison_pill() - if exc is None: - raise ConnectionClosedError('The connection was closed') - else: + self.dispatcher.dispatch_all(frames.PoisonPillFrame()) + if exc is not None: raise ConnectionLostError('The connection was unexpectedly lost') from exc - def _send_connection_closed_poison_pill(self): - frame = frames.PoisonPillFrame() - # send the poison pill to all open queues - self.dispatcher.dispatch_all(frame) - class FrameReader(object): def __init__(self): diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 6af4432..b788dc6 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -21,6 +21,7 @@ def remove_writer(self, channel_id): def dispatch(self, frame): if isinstance(frame, frames.HeartbeatFrame): return + # i think it makes sense to move this to Actor if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): return writer = self.queue_writers[frame.channel_id] diff --git a/test/integration_tests.py b/test/integration_tests.py index 8bdfa55..d1b633c 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -225,28 +225,25 @@ def it_should_not_throw(self): assert self.exception is None -class WhenAConnectionIsClosed: - def given_an_exception_handler_and_connection(self): - self.loop = asyncio.get_event_loop() - self.connection_closed_error_raised = False - self.loop.set_exception_handler(self.exception_handler) - self.connection = self.loop.run_until_complete(asynqp.connect()) +# class WhenAConnectionIsClosed: +# def given_an_exception_handler_and_connection(self): +# self.loop = asyncio.get_event_loop() +# self.connection_closed_error_raised = False +# self.loop.set_exception_handler(self.exception_handler) +# self.connection = self.loop.run_until_complete(asynqp.connect()) - def exception_handler(self, loop, context): - exception = context.get('exception') - if type(exception) is asynqp.exceptions.ConnectionClosedError: - self.connection_closed_error_raised = True - else: - self.loop.default_exception_handler(context) +# def exception_handler(self, loop, context): +# self.exception = context.get('exception') +# self.loop.default_exception_handler(context) - def when_the_connection_is_closed(self): - self.loop.run_until_complete(self.connection.close()) +# def when_the_connection_is_closed(self): +# self.loop.run_until_complete(self.connection.close()) - def it_should_raise_a_connection_closed_error(self): - assert self.connection_closed_error_raised is True +# def it_should_raise_a_connection_closed_error(self): +# assert self.exception is None - def cleanup(self): - self.loop.set_exception_handler(testing_exception_handler) +# def cleanup(self): +# self.loop.set_exception_handler(testing_exception_handler) class WhenAConnectionIsLost: diff --git a/test/protocol_tests.py b/test/protocol_tests.py index fb5c35e..e9ef554 100644 --- a/test/protocol_tests.py +++ b/test/protocol_tests.py @@ -3,7 +3,7 @@ import asynqp from asynqp import spec from asynqp import protocol -from asynqp.exceptions import ConnectionClosedError, ConnectionLostError +from asynqp.exceptions import ConnectionLostError from .base_contexts import MockDispatcherContext, MockServerContext from .util import testing_exception_handler @@ -134,29 +134,6 @@ def it_should_dispatch_the_method_twice(self): self.dispatcher.dispatch.assert_has_calls([mock.call(self.expected_frame), mock.call(self.expected_frame)]) -class WhenTheConnectionIsClosed(MockServerContext): - def given_an_exception_handler(self): - self.connection_closed_error_raised = False - self.loop.set_exception_handler(self.exception_handler) - - def exception_handler(self, loop, context): - exception = context.get('exception') - if type(exception) is ConnectionClosedError: - self.connection_closed_error_raised = True - else: - self.loop.default_exception_handler(context) - - def when_the_connection_is_closed(self): - self.loop.call_soon(self.protocol.connection_lost, None) - self.tick() - - def it_should_raise_a_connection_closed_error(self): - assert self.connection_closed_error_raised is True - - def cleanup(self): - self.loop.set_exception_handler(testing_exception_handler) - - class WhenTheConnectionIsLost(MockServerContext): def given_an_exception_handler(self): self.connection_lost_error_raised = False diff --git a/test/util.py b/test/util.py index b401766..0821a49 100644 --- a/test/util.py +++ b/test/util.py @@ -3,7 +3,7 @@ from unittest import mock import asynqp.frames from asynqp import protocol -from asynqp.exceptions import ConnectionClosedError +from asynqp.exceptions import ConnectionLostError def testing_exception_handler(loop, context): @@ -12,7 +12,7 @@ def testing_exception_handler(loop, context): ``ConnectionLostErros`` during tests ''' exception = context.get('exception') - if exception and isinstance(exception, ConnectionClosedError): + if exception and isinstance(exception, ConnectionLostError): pass else: loop.default_exception_handler(context) From 7a8972f228bf825b1ea0b7f0b04f7221c720f852 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 8 Jul 2015 17:10:30 +0100 Subject: [PATCH 041/118] refactoring around closing connections --- src/asynqp/bases.py | 24 ------------------------ src/asynqp/channel.py | 6 ++---- src/asynqp/connection.py | 15 +++------------ src/asynqp/frames.py | 4 ++-- src/asynqp/protocol.py | 2 +- src/asynqp/routing.py | 33 +++++++++++++++++++++++++++++---- test/base_contexts.py | 32 -------------------------------- test/connection_tests.py | 36 ++++++++++++------------------------ 8 files changed, 49 insertions(+), 103 deletions(-) delete mode 100644 src/asynqp/bases.py diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py deleted file mode 100644 index 2ea7c5e..0000000 --- a/src/asynqp/bases.py +++ /dev/null @@ -1,24 +0,0 @@ -class Sender(object): - def __init__(self, channel_id, protocol): - self.channel_id = channel_id - self.protocol = protocol - - def send_method(self, method): - self.protocol.send_method(self.channel_id, method) - - -class Actor(object): - def __init__(self, synchroniser, sender): - self.synchroniser = synchroniser - self.sender = sender - - def handle(self, frame): - try: - meth = getattr(self, 'handle_' + type(frame).__name__) - except AttributeError: - meth = getattr(self, 'handle_' + type(frame.payload).__name__) - - meth(frame) - - def handle_PoisonPillFrame(self, frame): - self.synchroniser.killall(ConnectionError) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 9bc1919..d58a7fe 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -1,6 +1,5 @@ import asyncio import re -from . import bases from . import frames from . import spec from . import queue @@ -185,7 +184,7 @@ def open(self): return channel -class ChannelActor(bases.Actor): +class ChannelActor(routing.Actor): def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) @@ -326,8 +325,7 @@ def receive_body(self, frame): self.reader.ready() -# basically just a collection of aliases with some arguments hard coded for convenience -class ChannelMethodSender(bases.Sender): +class ChannelMethodSender(routing.Sender): def __init__(self, channel_id, protocol, connection_info): super().__init__(channel_id, protocol) self.connection_info = connection_info diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 226e0d2..b92d443 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -1,7 +1,6 @@ import asyncio import sys from . import channel -from . import bases from . import spec from . import routing @@ -47,13 +46,6 @@ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) self.connection_info = connection_info - # this is ugly. when the connection is closing, all methods other than ConnectionCloseOK - # should be ignored. at the moment this behaviour is part of the dispatcher - # but this introduces an extra dependency between Connection and ConnectionActor which - # i don't like - self.closing = asyncio.Future() - self.closing.add_done_callback(lambda fut: dispatcher.closing.set_result(fut.result())) - @asyncio.coroutine def open_channel(self): """ @@ -73,7 +65,6 @@ def close(self): This method is a :ref:`coroutine `. """ - self.closing.set_result(True) self.sender.send_Close(0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) @@ -122,7 +113,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): return connection -class ConnectionActor(bases.Actor): +class ConnectionActor(routing.Actor): def __init__(self, synchroniser, sender, protocol, connection): super().__init__(synchroniser, sender) self.protocol = protocol @@ -138,7 +129,7 @@ def handle_ConnectionOpenOK(self, frame): self.synchroniser.notify(spec.ConnectionOpenOK) def handle_ConnectionClose(self, frame): - self.connection.closing.set_result(True) + self.closing.set_result(True) self.sender.send_CloseOK() self.protocol.transport.close() @@ -147,7 +138,7 @@ def handle_ConnectionCloseOK(self, frame): self.synchroniser.notify(spec.ConnectionCloseOK) -class ConnectionMethodSender(bases.Sender): +class ConnectionMethodSender(routing.Sender): def __init__(self, protocol): super().__init__(0, protocol) diff --git a/src/asynqp/frames.py b/src/asynqp/frames.py index 412e1d0..784d6ea 100644 --- a/src/asynqp/frames.py +++ b/src/asynqp/frames.py @@ -70,5 +70,5 @@ class PoisonPillFrame(Frame): channel_id = 0 payload = b'' - def __init__(self): - pass + def __init__(self, exception): + self.exception = exception diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 40833e2..9a082c1 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -46,7 +46,7 @@ def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) def connection_lost(self, exc): - self.dispatcher.dispatch_all(frames.PoisonPillFrame()) + self.dispatcher.dispatch_all(frames.PoisonPillFrame(exc)) if exc is not None: raise ConnectionLostError('The connection was unexpectedly lost') from exc diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index b788dc6..03b05b1 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -10,7 +10,6 @@ class Dispatcher(object): def __init__(self): self.queue_writers = {} - self.closing = asyncio.Future() def add_writer(self, channel_id, writer): self.queue_writers[channel_id] = writer @@ -21,9 +20,6 @@ def remove_writer(self, channel_id): def dispatch(self, frame): if isinstance(frame, frames.HeartbeatFrame): return - # i think it makes sense to move this to Actor - if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): - return writer = self.queue_writers[frame.channel_id] writer.enqueue(frame) @@ -32,6 +28,35 @@ def dispatch_all(self, frame): writer.enqueue(frame) +class Sender(object): + def __init__(self, channel_id, protocol): + self.channel_id = channel_id + self.protocol = protocol + + def send_method(self, method): + self.protocol.send_method(self.channel_id, method) + + +class Actor(object): + def __init__(self, synchroniser, sender): + self.synchroniser = synchroniser + self.sender = sender + self.closing = asyncio.Future() + + def handle(self, frame): + if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): + return + try: + meth = getattr(self, 'handle_' + type(frame).__name__) + except AttributeError: + meth = getattr(self, 'handle_' + type(frame.payload).__name__) + + meth(frame) + + def handle_PoisonPillFrame(self, frame): + self.synchroniser.killall(ConnectionError) + + class Synchroniser(object): _blocking_methods = set((spec.BasicCancelOK, # Consumer.cancel spec.ChannelCloseOK, # Channel.close diff --git a/test/base_contexts.py b/test/base_contexts.py index 30342f2..42774e2 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -114,38 +114,6 @@ def given_an_event_loop(self): self.loop = mock.Mock(spec=asyncio.AbstractEventLoop) -class LegacyConnectionContext(LoopContext): - def given_the_pieces_i_need_for_a_connection(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.protocol.transport = mock.Mock() - self.protocol.send_frame._is_coroutine = False # :( - - self.dispatcher = asynqp.routing.Dispatcher() - self.connection_info = ConnectionInfo('guest', 'guest', '/') - - -class LegacyOpenConnectionContext(LegacyConnectionContext): - def given_an_open_connection(self): - task = asyncio.async(open_connection(self.loop, self.protocol.transport, self.protocol, self.dispatcher, self.connection_info)) - self.tick() - - start_frame = asynqp.frames.MethodFrame(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) - self.dispatcher.dispatch(start_frame) - self.tick() - - self.frame_max = 131072 - tune_frame = asynqp.frames.MethodFrame(0, spec.ConnectionTune(0, self.frame_max, 600)) - self.dispatcher.dispatch(tune_frame) - self.tick() - - open_ok_frame = asynqp.frames.MethodFrame(0, spec.ConnectionOpenOK('')) - self.dispatcher.dispatch(open_ok_frame) - self.tick() - - self.protocol.reset_mock() - self.connection = task.result() - - class ProtocolContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) diff --git a/test/connection_tests.py b/test/connection_tests.py index 87613ba..bef997f 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,10 +1,8 @@ import asyncio import sys -import asynqp -from unittest import mock from asynqp import spec from asynqp.connection import open_connection, ConnectionInfo -from .base_contexts import LegacyOpenConnectionContext, MockServerContext, OpenConnectionContext +from .base_contexts import MockServerContext, OpenConnectionContext class WhenRespondingToConnectionStart(MockServerContext): @@ -69,40 +67,30 @@ def it_should_close_the_transport(self): assert self.transport.closed -# TODO: rewrite me to use a handler, not a queue writer -class WhenAConnectionThatIsClosingReceivesAMethod(LegacyOpenConnectionContext): +class WhenAConnectionThatIsClosingReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): t = asyncio.async(self.connection.close()) t._log_destroy_pending = False self.tick() - - start_method = spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US') - self.start_frame = asynqp.frames.MethodFrame(0, start_method) - self.mock_writer = mock.Mock() + self.server.reset() def when_another_frame_arrives(self): - with mock.patch.dict(self.dispatcher.queue_writers, {0: self.mock_writer}): - self.dispatcher.dispatch(self.start_frame) - self.tick() + self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) + self.tick() def it_MUST_be_discarded(self): - assert not self.mock_writer.method_calls + self.server.should_not_have_received_any() -# TODO: rewrite so it doesn't know about dispatcher -class WhenAConnectionThatWasClosedByTheServerReceivesAMethod(LegacyOpenConnectionContext): +class WhenAConnectionThatWasClosedByTheServerReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): - close_frame = asynqp.frames.MethodFrame(0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) - self.dispatcher.dispatch(close_frame) + self.server.send_method(0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) self.tick() - self.mock_writer = mock.Mock() + self.server.reset() def when_another_frame_arrives(self): - unexpected_frame = asynqp.frames.MethodFrame(1, spec.BasicDeliver('', 1, False, '', '')) - - with mock.patch.dict(self.dispatcher.queue_writers, {1: self.mock_writer}): - self.dispatcher.dispatch(unexpected_frame) - self.tick() + self.server.send_method(0, spec.BasicDeliver('', 1, False, '', '')) + self.tick() def it_MUST_be_discarded(self): - assert not self.mock_writer.method_calls + self.server.should_not_have_received_any() From c0f88bf703a85b8085ebe4fbd1d2141e9a01507e Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 8 Jul 2015 17:11:07 +0100 Subject: [PATCH 042/118] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01fcc01..7d64f12 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ asynqp ====== [![Build Status](https://travis-ci.org/benjamin-hodgson/asynqp.svg?branch=master)](https://travis-ci.org/benjamin-hodgson/asynqp) -[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.3)](https://readthedocs.org/projects/asynqp/?badge=v0.3) +[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.3)](https://readthedocs.org/projects/asynqp/?badge=v0.4) `asynqp` is an AMQP (aka [RabbitMQ](rabbitmq.com)) client library for Python 3.4's new [`asyncio`](https://docs.python.org/3.4/library/asyncio.html) module. From c93690fa61f5ab17009a2924be8ffebfaa08f278 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 8 Jul 2015 17:21:34 +0100 Subject: [PATCH 043/118] #31 Add Connection.closed future (which was documented but did not exist...) Also replaced ConnectionInfo with a dict because that's what it is really --- src/asynqp/__init__.py | 4 ++-- src/asynqp/channel.py | 2 +- src/asynqp/connection.py | 21 +++++++++------------ test/base_contexts.py | 4 ++-- test/connection_tests.py | 12 +++++++++--- test/integration_tests.py | 4 ++-- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index b7f86f9..027fa55 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -44,7 +44,7 @@ def connect(host='localhost', """ from .protocol import AMQP from .routing import Dispatcher - from .connection import ConnectionInfo, open_connection + from .connection import open_connection loop = asyncio.get_event_loop() if loop is None else loop @@ -57,7 +57,7 @@ def connect(host='localhost', dispatcher = Dispatcher() transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) - connection = yield from open_connection(loop, transport, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) + connection = yield from open_connection(loop, transport, protocol, dispatcher, {'username': username, 'password': password, 'virtual_host': virtual_host}) return connection diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index d58a7fe..c320274 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -387,7 +387,7 @@ def send_content(self, msg): header_frame = frames.ContentHeaderFrame(self.channel_id, header_payload) self.protocol.send_frame(header_frame) - for payload in message.get_frame_payloads(msg, self.connection_info.frame_max - 8): + for payload in message.get_frame_payloads(msg, self.connection_info['frame_max'] - 8): frame = frames.ContentBodyFrame(self.channel_id, payload) self.protocol.send_frame(frame) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index b92d443..4f4a992 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -5,13 +5,6 @@ from . import routing -class ConnectionInfo(object): - def __init__(self, username, password, virtual_host): - self.username = username - self.password = password - self.virtual_host = virtual_host - - class Connection(object): """ Manage connections to AMQP brokers. @@ -39,13 +32,15 @@ class Connection(object): The :class:`~asyncio.Protocol` which is paired with the transport """ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): - self.transport = transport - self.protocol = protocol self.synchroniser = synchroniser self.sender = sender self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) self.connection_info = connection_info + self.transport = transport + self.protocol = protocol + self.closed = asyncio.Future() + @asyncio.coroutine def open_channel(self): """ @@ -67,6 +62,7 @@ def close(self): """ self.sender.send_Close(0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) + self.closed.set_result(True) @asyncio.coroutine @@ -90,18 +86,18 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): "version": "0.1", # todo: use pkg_resources to inspect the package "platform": sys.version}, 'AMQPLAIN', - {'LOGIN': connection_info.username, 'PASSWORD': connection_info.password}, + {'LOGIN': connection_info['username'], 'PASSWORD': connection_info['password']}, 'en_US' ) reader.ready() frame = yield from synchroniser.await(spec.ConnectionTune) # just agree with whatever the server wants. Make this configurable in future - connection_info.frame_max = frame.payload.frame_max + connection_info['frame_max'] = frame.payload.frame_max heartbeat_interval = frame.payload.heartbeat sender.send_TuneOK(frame.payload.channel_max, frame.payload.frame_max, heartbeat_interval) - sender.send_Open(connection_info.virtual_host) + sender.send_Open(connection_info['virtual_host']) protocol.start_heartbeat(heartbeat_interval) reader.ready() @@ -132,6 +128,7 @@ def handle_ConnectionClose(self, frame): self.closing.set_result(True) self.sender.send_CloseOK() self.protocol.transport.close() + self.connection.closed.set_result(True) def handle_ConnectionCloseOK(self, frame): self.protocol.transport.close() diff --git a/test/base_contexts.py b/test/base_contexts.py index 42774e2..c2bfb8d 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -4,7 +4,7 @@ from asyncio import test_utils from asynqp import spec from asynqp import protocol -from asynqp.connection import ConnectionInfo, open_connection +from asynqp.connection import open_connection from unittest import mock from .util import MockServer, FakeTransport @@ -42,7 +42,7 @@ def given_a_mock_server_on_the_other_end_of_the_transport(self): class OpenConnectionContext(MockServerContext): def given_an_open_connection(self): - connection_info = ConnectionInfo('guest', 'guest', '/') + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} task = asyncio.async(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.tick() diff --git a/test/connection_tests.py b/test/connection_tests.py index bef997f..2de8300 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,13 +1,13 @@ import asyncio import sys from asynqp import spec -from asynqp.connection import open_connection, ConnectionInfo +from asynqp.connection import open_connection from .base_contexts import MockServerContext, OpenConnectionContext class WhenRespondingToConnectionStart(MockServerContext): def given_I_wrote_the_protocol_header(self): - connection_info = ConnectionInfo('guest', 'guest', '/') + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) def when_ConnectionStart_arrives(self): @@ -25,7 +25,7 @@ def it_should_send_start_ok(self): class WhenRespondingToConnectionTune(MockServerContext): def given_a_started_connection(self): - connection_info = ConnectionInfo('guest', 'guest', '/') + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) @@ -45,6 +45,9 @@ def when_the_close_frame_arrives(self): def it_should_send_close_ok(self): self.server.should_have_received_method(0, spec.ConnectionCloseOK()) + def it_should_set_the_future(self): + assert self.connection.closed.done() + class WhenTheApplicationClosesTheConnection(OpenConnectionContext): def when_I_close_the_connection(self): @@ -66,6 +69,9 @@ def when_connection_close_ok_arrives(self): def it_should_close_the_transport(self): assert self.transport.closed + def it_should_set_the_future(self): + assert self.connection.closed.done() + class WhenAConnectionThatIsClosingReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): diff --git a/test/integration_tests.py b/test/integration_tests.py index d1b633c..9d07ebe 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -178,7 +178,7 @@ def start_consumer(self): class WhenPublishingAndGettingALongMessage(BoundQueueContext): def given_a_multi_frame_message_and_a_consumer(self): - frame_max = self.connection.connection_info.frame_max + frame_max = self.connection.connection_info['frame_max'] body1 = "a" * (frame_max - 8) body2 = "b" * (frame_max - 8) body3 = "c" * (frame_max - 8) @@ -195,7 +195,7 @@ def it_should_return_my_message(self): class WhenPublishingAndConsumingALongMessage(BoundQueueContext): def given_a_multi_frame_message(self): - frame_max = self.connection.connection_info.frame_max + frame_max = self.connection.connection_info['frame_max'] body1 = "a" * (frame_max - 8) body2 = "b" * (frame_max - 8) body3 = "c" * (frame_max - 8) From 3fd2024ca4ac5df07ae32fe93d9dae4fb6f700c6 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 26 Jul 2015 13:16:10 +0100 Subject: [PATCH 044/118] Upgrade to new, faster Travis infrastructure --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 488a61a..7123fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +sudo: false + language: python python: 3.4 From b5374dd211202211033bf7ced82cfbbbfdd0695a Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 26 Jul 2015 13:16:54 +0100 Subject: [PATCH 045/118] fix badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d64f12..b5748bc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ asynqp ====== [![Build Status](https://travis-ci.org/benjamin-hodgson/asynqp.svg?branch=master)](https://travis-ci.org/benjamin-hodgson/asynqp) -[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.3)](https://readthedocs.org/projects/asynqp/?badge=v0.4) +[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.4)](https://readthedocs.org/projects/asynqp/?badge=v0.4) `asynqp` is an AMQP (aka [RabbitMQ](rabbitmq.com)) client library for Python 3.4's new [`asyncio`](https://docs.python.org/3.4/library/asyncio.html) module. From 728dc9fab80c29de307a2905202bb6a26694a758 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 24 Aug 2015 11:36:01 -0400 Subject: [PATCH 046/118] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7123fa2..ca8b3f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: script: - flake8 src test --ignore=E501 - run-contexts -v - - cd doc && make html + - pushd doc && make html && popd deploy: provider: pypi From 3a23f7186c112c509d7d997285f2bf599be7f218 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 21:13:02 +0100 Subject: [PATCH 047/118] Create requirements.txt --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7578542 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +contexts +colorama +sphinx +flake8 From 7be77970a6ab1df4e2ac6605c9474ae1c92c098f Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 21:13:18 +0100 Subject: [PATCH 048/118] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ca8b3f7..476af18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: 3.4 services: rabbitmq install: - - pip install contexts colorama sphinx flake8 + - pip install -r requirements.txt - python setup.py install script: From 7057da268f83e89204bb446ee189ea3b1ed61ea4 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 21:21:30 +0100 Subject: [PATCH 049/118] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b5748bc..e0d3c6f 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ asynqp [![Build Status](https://travis-ci.org/benjamin-hodgson/asynqp.svg?branch=master)](https://travis-ci.org/benjamin-hodgson/asynqp) [![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.4)](https://readthedocs.org/projects/asynqp/?badge=v0.4) +[![Requirements Status](https://requires.io/github/benjamin-hodgson/asynqp/requirements.svg?branch=master)](https://requires.io/github/benjamin-hodgson/asynqp/requirements/?branch=master) `asynqp` is an AMQP (aka [RabbitMQ](rabbitmq.com)) client library for Python 3.4's new [`asyncio`](https://docs.python.org/3.4/library/asyncio.html) module. From df5aacdb24e5380d88f2e85759d31a20708bcb6f Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 23:35:47 +0100 Subject: [PATCH 050/118] coveralls --- .travis.yml | 8 ++++---- README.md | 1 + requirements.txt | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 476af18..3109eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,12 @@ install: script: - flake8 src test --ignore=E501 - - run-contexts -v + - coverage run --source=asynqp -m contexts - pushd doc && make html && popd +after_success: + coveralls + deploy: provider: pypi user: benjamin.hodgson @@ -24,6 +27,3 @@ deploy: tags: true all_branches: true distributions: "sdist bdist_wheel bdist_egg" - -after_deploy: - - curl -X POST http://readthedocs.org/build/asynqp diff --git a/README.md b/README.md index e0d3c6f..3e5f1b9 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ asynqp [![Build Status](https://travis-ci.org/benjamin-hodgson/asynqp.svg?branch=master)](https://travis-ci.org/benjamin-hodgson/asynqp) [![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.4)](https://readthedocs.org/projects/asynqp/?badge=v0.4) +[![Coverage Status](https://coveralls.io/repos/benjamin-hodgson/asynqp/badge.svg?branch=master&service=github)](https://coveralls.io/github/benjamin-hodgson/asynqp?branch=master) [![Requirements Status](https://requires.io/github/benjamin-hodgson/asynqp/requirements.svg?branch=master)](https://requires.io/github/benjamin-hodgson/asynqp/requirements/?branch=master) `asynqp` is an AMQP (aka [RabbitMQ](rabbitmq.com)) client library for diff --git a/requirements.txt b/requirements.txt index 7578542..335d90c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ contexts colorama sphinx flake8 +coverage +coveralls From df640058e52fd3042b2d0b81e30054897bb27c80 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 23:36:59 +0100 Subject: [PATCH 051/118] woops --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3109eb2..d4ce3a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: script: - flake8 src test --ignore=E501 - - coverage run --source=asynqp -m contexts + - coverage run --source=asynqp -m contexts -v - pushd doc && make html && popd after_success: From 580010798289953bc4112823eceef2c75f7a1c3f Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 4 Sep 2015 23:37:34 +0100 Subject: [PATCH 052/118] coverage --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4ce3a2..c964286 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,11 @@ services: rabbitmq install: - pip install -r requirements.txt - - python setup.py install + - python setup.py develop script: - flake8 src test --ignore=E501 - - coverage run --source=asynqp -m contexts -v + - coverage run --source=src -m contexts -v - pushd doc && make html && popd after_success: From d3ead4716cb0bba956beb86902437b86ae95b49b Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 15 Sep 2015 22:33:14 +0100 Subject: [PATCH 053/118] Fix #37. You should be able to open multiple channels concurrently --- src/asynqp/channel.py | 20 ++++++++++++++------ test/channel_tests.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index c320274..7581d2d 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -152,13 +152,15 @@ def __init__(self, loop, protocol, dispatcher, connection_info): self.protocol = protocol self.dispatcher = dispatcher self.connection_info = connection_info - self.next_channel_id = 1 + self.next_channel_id = 0 @asyncio.coroutine def open(self): + self.next_channel_id += 1 + channel_id = self.next_channel_id synchroniser = routing.Synchroniser() - sender = ChannelMethodSender(self.next_channel_id, self.protocol, self.connection_info) + sender = ChannelMethodSender(channel_id, self.protocol, self.connection_info) basic_return_consumer = BasicReturnConsumer() consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) @@ -168,18 +170,24 @@ def open(self): handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) queue_factory = queue.QueueFactory(sender, synchroniser, reader, consumers) - channel = Channel(self.next_channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader) + channel = Channel(channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader) - self.dispatcher.add_writer(self.next_channel_id, writer) + self.dispatcher.add_writer(channel_id, writer) try: sender.send_ChannelOpen() reader.ready() yield from synchroniser.await(spec.ChannelOpenOK) except: - self.dispatcher.remove_writer(self.next_channel_id) + # don't rollback self.next_channel_id; + # another call may have entered this method + # concurrently while we were yielding + # and we don't want to end up with duplicate channels. + # If we leave self.next_channel_id incremented, the worst + # that happens is we end up with non-sequential channel numbers. + # Small price to pay to keep this method re-entrant. + self.dispatcher.remove_writer(channel_id) raise - self.next_channel_id += 1 reader.ready() return channel diff --git a/test/channel_tests.py b/test/channel_tests.py index 1af4024..45ce5ad 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -16,6 +16,17 @@ def it_should_send_a_channel_open_frame(self): self.server.should_have_received_method(1, spec.ChannelOpen('')) +class WhenOpeningMultipleChannelsConcurrently(OpenConnectionContext): + def when_the_user_wants_to_open_several_channels(self): + self.async_partial(asyncio.wait([self.connection.open_channel(), self.connection.open_channel()])) + + def it_should_send_a_channel_open_frame_for_channel_1(self): + self.server.should_have_received_method(1, spec.ChannelOpen('')) + + def it_should_send_a_channel_open_frame_for_channel_2(self): + self.server.should_have_received_method(2, spec.ChannelOpen('')) + + class WhenChannelOpenOKArrives(OpenConnectionContext): def given_the_user_has_called_open_channel(self): self.task = asyncio.async(self.connection.open_channel()) From e30c0fda292e5b2a55c685828d525c4fcd5cee26 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 15 Sep 2015 23:02:50 +0100 Subject: [PATCH 054/118] #34 build API for handling failures in a consumer --- src/asynqp/channel.py | 10 +++++++- src/asynqp/queue.py | 15 ++++++++++++ test/queue_tests.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 7581d2d..927070d 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -165,7 +165,7 @@ def open(self): consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - handler = ChannelActor(synchroniser, sender) + handler = ChannelActor(consumers, synchroniser, sender) reader, writer = routing.create_reader_and_writer(handler) handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) @@ -193,6 +193,14 @@ def open(self): class ChannelActor(routing.Actor): + def __init__(self, consumers, *args, **kwargs): + super().__init__(*args, **kwargs) + self.consumers = consumers + + def handle_PoisonPillFrame(self, frame): + super().handle_PoisonPillFrame(frame) + self.consumers.error(frame.exception) + def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index d094c7d..c90eea8 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -81,6 +81,14 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, ar Start a consumer on the queue. Messages will be delivered asynchronously to the consumer. The callback function will be called whenever a new message arrives on the queue. + Advanced usage: the callback object must be callable + (it must be a function or define a ``__call__`` method), + but may also define some further methods: + + * ``callback.on_cancel()``: called with no parameters when the consumer is successfully cancelled. + * ``callback.on_error(exc)``: called when the channel is closed due to an error. + The argument passed is the exception which caused the error. + This method is a :ref:`coroutine `. :param callable callback: a callback to be called when a message is delivered. @@ -250,6 +258,8 @@ def cancel(self): yield from self.synchroniser.await(spec.BasicCancelOK) self.cancelled = True self.cancelled_future.set_result(self) + if hasattr(self.callback, 'on_cancel'): + self.callback.on_cancel() self.reader.ready() @@ -288,3 +298,8 @@ def deliver(self, tag, msg): assert tag in self.consumers, "Message got delivered to a non existent consumer" consumer = self.consumers[tag] self.loop.call_soon(consumer.callback, msg) + + def error(self, exc): + for consumer in self.consumers.values(): + if hasattr(consumer.callback, 'on_error'): + consumer.callback.on_error(exc) diff --git a/test/queue_tests.py b/test/queue_tests.py index 6f8ba32..e367aba 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -1,5 +1,6 @@ import asyncio from datetime import datetime +import contexts import asynqp from asynqp import message from asynqp import frames @@ -267,6 +268,60 @@ def it_should_be_cancelled(self): assert self.consumer.cancelled +class WhenCancelOKArrivesForAConsumerWithAnOnCancelMethod(QueueContext): + def given_I_started_and_cancelled_a_consumer(self): + self.consumer = self.ConsumerWithOnCancel() + task = asyncio.async(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + asyncio.async(task.result().cancel()) + self.tick() + + def when_BasicCancelOK_arrives(self): + self.server.send_method(self.channel.id, spec.BasicCancelOK('made.up.tag')) + + def it_should_call_on_cancel(self): + assert self.consumer.on_cancel_called + + class ConsumerWithOnCancel: + def __init__(self): + self.on_cancel_called = False + + def __call__(self): + pass + + def on_cancel(self): + self.on_cancel_called = True + + +class WhenAConsumerWithAnOnCancelMethodIsKilledDueToAnError(QueueContext): + def given_I_started_a_consumer(self): + self.consumer = self.ConsumerWithOnError() + asyncio.async(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.exception = Exception() + + def when_the_connection_dies(self): + contexts.catch(self.protocol.connection_lost, self.exception) + self.tick() + + def it_should_call_on_error(self): + assert self.consumer.exc is self.exception + + class ConsumerWithOnError: + def __init__(self): + self.exc = None + + def __call__(self): + pass + + def on_error(self, exc): + self.exc = exc + + class WhenIPurgeAQueue(QueueContext): def because_I_purge_the_queue(self): self.async_partial(self.queue.purge()) From 10ce858d699bda8ec2a089a60b0bd3d40d45b244 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 15 Sep 2015 23:19:20 +0100 Subject: [PATCH 055/118] Fix #32 --- src/asynqp/serialisation.py | 60 ++++++++++++++++++------------------- test/serialisation_tests.py | 2 ++ 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index c887024..694a369 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -102,6 +102,7 @@ def qpid_rabbit_mq_table(): b's': _read_short, b'I': _read_long, b'l': _read_long_long, + b'f': _read_float, b'S': _read_long_string, b'A': _read_array, b'V': _read_void, @@ -193,6 +194,11 @@ def _read_unsigned_long_long(stream): return x, 8 +def _read_float(stream): + x, = struct.unpack('!f', stream.read(4)) + return x, 4 + + def _read_timestamp(stream): x, = struct.unpack('!Q', stream.read(8)) # From datetime.fromutctimestamp converts it to a local timestamp without timezone information @@ -241,38 +247,28 @@ def pack_long_string(string): def pack_field_value(value): - buffer = b'' + if value is None: + return b'V' if isinstance(value, bool): - buffer += b't' - buffer += pack_bool(value) - elif isinstance(value, dict): - buffer += b'F' - buffer += pack_table(value) - elif isinstance(value, list): - buffer += b'A' - buffer += pack_array(value) - elif isinstance(value, bytes): - buffer += b'x' - buffer += pack_byte_array(value) - elif isinstance(value, str): - buffer += b'S' - buffer += pack_long_string(value) - elif isinstance(value, datetime): - buffer += b'T' - buffer += pack_timestamp(value) - elif isinstance(value, int): + return b't' + pack_bool(value) + if isinstance(value, dict): + return b'F' + pack_table(value) + if isinstance(value, list): + return b'A' + pack_array(value) + if isinstance(value, bytes): + return b'x' + pack_byte_array(value) + if isinstance(value, str): + return b'S' + pack_long_string(value) + if isinstance(value, datetime): + return b'T' + pack_timestamp(value) + if isinstance(value, int): if value.bit_length() < 8: - buffer += b'b' - buffer += pack_signed_byte(value) - elif value.bit_length() < 32: - buffer += b'I' - buffer += pack_long(value) - else: - raise NotImplementedError() - else: - raise NotImplementedError() - - return buffer + return b'b' + pack_signed_byte(value) + if value.bit_length() < 32: + return b'I' + pack_long(value) + if isinstance(value, float): + return b'f' + pack_float(value) + raise NotImplementedError() def pack_table(d): @@ -321,6 +317,10 @@ def pack_unsigned_long_long(number): return struct.pack('!Q', number) +def pack_float(number): + return struct.pack('!f', number) + + def pack_bool(b): return struct.pack('!?', b) diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index b75685d..1b674ca 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -35,6 +35,8 @@ def examples_of_tables(cls): yield {'f': -0x7FFFFFFF, 'g': 0x7FFFFFFF} yield {'x': b"\x01\x02"} yield {'x': []} + yield {'l': None} + yield {'l': 1.0} def because_we_pack_and_unpack_the_table(self, table): self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) From c9d76b460826d343db51395918c9c4f74fbaa832 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 16 Sep 2015 00:57:53 +0100 Subject: [PATCH 056/118] test on python 3.5 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c964286..39956d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ sudo: false language: python -python: 3.4 +python: + - 3.4 + - 3.5 services: rabbitmq From 82815f7b19fe05a3fcd677d6e0ea7a25528809b3 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 16 Sep 2015 01:32:56 +0100 Subject: [PATCH 057/118] setup.py 3.5 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c82d52c..461da0e 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ "Development Status :: 3 - Alpha", "Programming Language :: Python", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", "License :: OSI Approved :: MIT License", "Intended Audience :: Developers", "Intended Audience :: Information Technology", From 28b0b59c839a1ba2d743f0996f32ea22bd93d58f Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Thu, 17 Sep 2015 15:32:18 +0100 Subject: [PATCH 058/118] fix #39 (docs were wrong) --- src/asynqp/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/asynqp/message.py b/src/asynqp/message.py index ff1c4ad..95a0b74 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -145,7 +145,7 @@ def reject(self, *, requeue=True): """ Reject the message. - :keyword bool redeliver: if true, the broker will attempt to requeue the + :keyword bool requeue: if true, the broker will attempt to requeue the message and deliver it to an alternate consumer. """ self.sender.send_BasicReject(self.delivery_tag, requeue) From b509f10cd2d5d0e25a85ebc35b181c7236ef2d51 Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 19 Sep 2015 15:23:30 +0300 Subject: [PATCH 059/118] Added explicit loop passing to all componencts from connection. --- src/asynqp/channel.py | 39 +++++++++++++++++++++++++-------------- src/asynqp/connection.py | 13 +++++++------ src/asynqp/queue.py | 25 +++++++++++++++++++------ src/asynqp/routing.py | 34 +++++++++++++++++++++++++--------- 4 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 927070d..694840a 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -28,7 +28,10 @@ class Channel(object): the numerical ID of the channel """ - def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader): + def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.id = id self.synchroniser = synchroniser self.sender = sender @@ -158,19 +161,22 @@ def __init__(self, loop, protocol, dispatcher, connection_info): def open(self): self.next_channel_id += 1 channel_id = self.next_channel_id - synchroniser = routing.Synchroniser() + synchroniser = routing.Synchroniser(loop=self.loop) sender = ChannelMethodSender(channel_id, self.protocol, self.connection_info) - basic_return_consumer = BasicReturnConsumer() + basic_return_consumer = BasicReturnConsumer(loop=self.loop) consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - handler = ChannelActor(consumers, synchroniser, sender) - reader, writer = routing.create_reader_and_writer(handler) + handler = ChannelActor(consumers, synchroniser, sender, loop=self.loop) + reader, writer = routing.create_reader_and_writer(handler, loop=self.loop) handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) - queue_factory = queue.QueueFactory(sender, synchroniser, reader, consumers) - channel = Channel(channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader) + queue_factory = queue.QueueFactory( + sender, synchroniser, reader, consumers, loop=self.loop) + channel = Channel( + channel_id, synchroniser, sender, basic_return_consumer, + queue_factory, reader, loop=self.loop) self.dispatcher.add_writer(channel_id, writer) try: @@ -229,7 +235,8 @@ def handle_BasicGetEmpty(self, frame): self.synchroniser.notify(spec.BasicGetEmpty, False) def handle_BasicGetOK(self, frame): - asyncio.async(self.message_receiver.receive_getOK(frame)) + assert self.message_receiver is not None, "message_receiver not set" + asyncio.async(self.message_receiver.receive_getOK(frame), loop=self._loop) def handle_BasicConsumeOK(self, frame): self.synchroniser.notify(spec.BasicConsumeOK, frame.payload.consumer_tag) @@ -238,13 +245,16 @@ def handle_BasicCancelOK(self, frame): self.synchroniser.notify(spec.BasicCancelOK) def handle_BasicDeliver(self, frame): - asyncio.async(self.message_receiver.receive_deliver(frame)) + assert self.message_receiver is not None, "message_receiver not set" + asyncio.async(self.message_receiver.receive_deliver(frame), loop=self._loop) def handle_ContentHeaderFrame(self, frame): - asyncio.async(self.message_receiver.receive_header(frame)) + assert self.message_receiver is not None, "message_receiver not set" + asyncio.async(self.message_receiver.receive_header(frame), loop=self._loop) def handle_ContentBodyFrame(self, frame): - asyncio.async(self.message_receiver.receive_body(frame)) + assert self.message_receiver is not None, "message_receiver not set" + asyncio.async(self.message_receiver.receive_body(frame), loop=self._loop) def handle_ChannelClose(self, frame): self.sender.send_CloseOK() @@ -258,7 +268,8 @@ def handle_BasicQosOK(self, frame): self.synchroniser.notify(spec.BasicQosOK) def handle_BasicReturn(self, frame): - asyncio.async(self.message_receiver.receive_return(frame)) + assert self.message_receiver is not None, "message_receiver not set" + asyncio.async(self.message_receiver.receive_return(frame), loop=self._loop) class MessageReceiver(object): @@ -411,9 +422,9 @@ def send_content(self, msg): class BasicReturnConsumer(object): tag = -1 # a 'real' tag is a string so there will never be a clash - def __init__(self): + def __init__(self, *, loop): self.callback = self.default_behaviour - self.cancelled_future = asyncio.Future() + self.cancelled_future = asyncio.Future(loop=loop) def set_callback(self, callback): if callback is None: diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 4f4a992..7f01fb8 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -32,6 +32,7 @@ class Connection(object): The :class:`~asyncio.Protocol` which is paired with the transport """ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): + self._loop = loop self.synchroniser = synchroniser self.sender = sender self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) @@ -39,7 +40,7 @@ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, self.transport = transport self.protocol = protocol - self.closed = asyncio.Future() + self.closed = asyncio.Future(loop=loop) @asyncio.coroutine def open_channel(self): @@ -67,13 +68,13 @@ def close(self): @asyncio.coroutine def open_connection(loop, transport, protocol, dispatcher, connection_info): - synchroniser = routing.Synchroniser() + synchroniser = routing.Synchroniser(loop=loop) sender = ConnectionMethodSender(protocol) connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) - handler = ConnectionActor(synchroniser, sender, protocol, connection) + handler = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) - reader, writer = routing.create_reader_and_writer(handler) + reader, writer = routing.create_reader_and_writer(handler, loop=loop) try: dispatcher.add_writer(0, writer) @@ -110,8 +111,8 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): class ConnectionActor(routing.Actor): - def __init__(self, synchroniser, sender, protocol, connection): - super().__init__(synchroniser, sender) + def __init__(self, synchroniser, sender, protocol, connection, *, loop=None): + super().__init__(synchroniser, sender, loop=loop) self.protocol = protocol self.connection = connection diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index c90eea8..7dc3cd3 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -37,7 +37,10 @@ class Queue(object): A dictionary of the extra arguments that were used to declare the queue. """ - def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments): + def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.reader = reader self.consumers = consumers self.synchroniser = synchroniser @@ -106,7 +109,9 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, ar self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive, arguments or {}) tag = yield from self.synchroniser.await(spec.BasicConsumeOK) - consumer = Consumer(tag, callback, self.sender, self.synchroniser, self.reader) + consumer = Consumer( + tag, callback, self.sender, self.synchroniser, self.reader, + loop=self._loop) self.consumers.add_consumer(consumer) self.reader.ready() return consumer @@ -238,14 +243,17 @@ class Consumer(object): Boolean. True if the consumer has been successfully cancelled. """ - def __init__(self, tag, callback, sender, synchroniser, reader): + def __init__(self, tag, callback, sender, synchroniser, reader, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.tag = tag self.callback = callback self.sender = sender self.cancelled = False self.synchroniser = synchroniser self.reader = reader - self.cancelled_future = asyncio.Future() + self.cancelled_future = asyncio.Future(loop=self._loop) @asyncio.coroutine def cancel(self): @@ -264,7 +272,10 @@ def cancel(self): class QueueFactory(object): - def __init__(self, sender, synchroniser, reader, consumers): + def __init__(self, sender, synchroniser, reader, consumers, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.sender = sender self.synchroniser = synchroniser self.reader = reader @@ -279,7 +290,9 @@ def declare(self, name, durable, exclusive, auto_delete, arguments): self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments) name = yield from self.synchroniser.await(spec.QueueDeclareOK) - q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete, arguments) + q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, + name, durable, exclusive, auto_delete, arguments, + loop=self._loop) self.reader.ready() return q diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 03b05b1..555888e 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -38,10 +38,13 @@ def send_method(self, method): class Actor(object): - def __init__(self, synchroniser, sender): + def __init__(self, synchroniser, sender, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.synchroniser = synchroniser self.sender = sender - self.closing = asyncio.Future() + self.closing = asyncio.Future(loop=self._loop) def handle(self, frame): if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): @@ -62,12 +65,15 @@ class Synchroniser(object): spec.ChannelCloseOK, # Channel.close spec.ConnectionCloseOK)) # Connection.close - def __init__(self): + def __init__(self, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self._futures = OrderedManyToManyMap() self.connection_closed = False def await(self, *expected_methods): - fut = asyncio.Future() + fut = asyncio.Future(loop=self._loop) if self.connection_closed: for method in expected_methods: @@ -103,9 +109,11 @@ def killall(self, exc): self._futures.remove_item(fut) -def create_reader_and_writer(handler): - q = asyncio.Queue() - reader = QueueReader(handler, q) +def create_reader_and_writer(handler, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + q = asyncio.Queue(loop=loop) + reader = QueueReader(handler, q, loop=loop) writer = QueueWriter(q) return reader, writer @@ -114,7 +122,10 @@ def create_reader_and_writer(handler): # When the frame does arrive, dispatch it to the handler and do nothing # until someone calls ready() again. class QueueReader(object): - def __init__(self, handler, q): + def __init__(self, handler, q, *, loop=None): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop self.handler = handler self.q = q self.is_waiting = False @@ -122,7 +133,12 @@ def __init__(self, handler, q): def ready(self): assert not self.is_waiting, "ready() got called while waiting for a frame to be read" self.is_waiting = True - t = asyncio.async(self._read_next()) + + # XXX: Refactor this. There should be only 1 async task per QueueReader + # It will read frames in a `while True:` loop and will be canceled + # when connection is closed. + + t = asyncio.async(self._read_next(), loop=self._loop) if _TEST: # this feels hacky to me t._log_destroy_pending = False From ea1fe9da2b7b1c97bd25a1c39e384940a4502c85 Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 19 Sep 2015 18:27:46 +0300 Subject: [PATCH 060/118] Removed unused optional asyncion.get_event_loop calls --- src/asynqp/channel.py | 4 +--- src/asynqp/queue.py | 12 +++--------- src/asynqp/routing.py | 12 +++--------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 694840a..2ece4d4 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -28,9 +28,7 @@ class Channel(object): the numerical ID of the channel """ - def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader, *, loop): self._loop = loop self.id = id self.synchroniser = synchroniser diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index 7dc3cd3..f47352d 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -37,9 +37,7 @@ class Queue(object): A dictionary of the extra arguments that were used to declare the queue. """ - def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments, *, loop): self._loop = loop self.reader = reader self.consumers = consumers @@ -243,9 +241,7 @@ class Consumer(object): Boolean. True if the consumer has been successfully cancelled. """ - def __init__(self, tag, callback, sender, synchroniser, reader, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, tag, callback, sender, synchroniser, reader, *, loop): self._loop = loop self.tag = tag self.callback = callback @@ -272,9 +268,7 @@ def cancel(self): class QueueFactory(object): - def __init__(self, sender, synchroniser, reader, consumers, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, sender, synchroniser, reader, consumers, *, loop): self._loop = loop self.sender = sender self.synchroniser = synchroniser diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 555888e..013a988 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -38,9 +38,7 @@ def send_method(self, method): class Actor(object): - def __init__(self, synchroniser, sender, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, synchroniser, sender, *, loop): self._loop = loop self.synchroniser = synchroniser self.sender = sender @@ -65,9 +63,7 @@ class Synchroniser(object): spec.ChannelCloseOK, # Channel.close spec.ConnectionCloseOK)) # Connection.close - def __init__(self, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, *, loop): self._loop = loop self._futures = OrderedManyToManyMap() self.connection_closed = False @@ -109,9 +105,7 @@ def killall(self, exc): self._futures.remove_item(fut) -def create_reader_and_writer(handler, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() +def create_reader_and_writer(handler, *, loop): q = asyncio.Queue(loop=loop) reader = QueueReader(handler, q, loop=loop) writer = QueueWriter(q) From 784de0b50f9a2fac23cc2517605e3fd83550047f Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 19 Sep 2015 18:30:37 +0300 Subject: [PATCH 061/118] And another unused get_event_loop call --- src/asynqp/routing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 013a988..c207901 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -116,9 +116,7 @@ def create_reader_and_writer(handler, *, loop): # When the frame does arrive, dispatch it to the handler and do nothing # until someone calls ready() again. class QueueReader(object): - def __init__(self, handler, q, *, loop=None): - if loop is None: - loop = asyncio.get_event_loop() + def __init__(self, handler, q, *, loop): self._loop = loop self.handler = handler self.q = q From 5522c22a40dec74920f2f637758516172b0932be Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 20 Sep 2015 22:44:38 +0300 Subject: [PATCH 062/118] Removed unneded local `_loop` veriable in Connection --- src/asynqp/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 7f01fb8..73367d7 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -32,7 +32,6 @@ class Connection(object): The :class:`~asyncio.Protocol` which is paired with the transport """ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): - self._loop = loop self.synchroniser = synchroniser self.sender = sender self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) From 4bb6999ae85d3ac467bf889e558664259652da28 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 21 Sep 2015 20:58:12 +0300 Subject: [PATCH 063/118] Added some tests on unset loop --- test/integration_tests.py | 81 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/test/integration_tests.py b/test/integration_tests.py index 9d07ebe..c55c87c 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -2,6 +2,7 @@ import asynqp import socket import contexts +from asyncio import test_utils from .util import testing_exception_handler @@ -332,3 +333,83 @@ def it_should_not_hang(self): def cleanup(self): asynqp.routing._TEST = False + + +class WhenPublishingWithUnsetLoop: + + def given_I_have_a_queue(self): + @asyncio.coroutine + def set_up(): + self.connection = yield from asynqp.connect(loop=self.loop) + self.channel = yield from self.connection.open_channel() + self.exchange = yield from self.channel.declare_exchange( + '', 'direct') + self.queue = yield from self.channel.declare_queue( + durable=False, + exclusive=True, + auto_delete=True) + self.loop = asyncio.get_event_loop() + asyncio.set_event_loop(None) + self.loop.run_until_complete(set_up()) + + def when_I_publish_the_message(self): + message = asynqp.Message(b"Test message") + self.exchange.publish(message, self.queue.name) + + def it_should_return_my_message(self): + for retry in range(10): + msg = self.loop.run_until_complete(self.queue.get(no_ack=True)) + if msg is not None: + break + assert msg.body == b"Test message" + + def cleanup_loop(self): + @asyncio.coroutine + def tear_down(): + yield from self.channel.close() + yield from self.connection.close() + self.loop.run_until_complete(tear_down()) + asyncio.set_event_loop(self.loop) + + +class WhenConsumingWithUnsetLoop: + + def given_I_published_a_message(self): + @asyncio.coroutine + def set_up(): + self.connection = yield from asynqp.connect(loop=self.loop) + self.channel = yield from self.connection.open_channel() + self.exchange = yield from self.channel.declare_exchange( + '', 'direct') + self.queue = yield from self.channel.declare_queue( + durable=False, + exclusive=True, + auto_delete=True) + self.loop = asyncio.get_event_loop() + asyncio.set_event_loop(None) + self.loop.run_until_complete(set_up()) + + message = asynqp.Message(b"Test message") + self.exchange.publish(message, self.queue.name) + + def when_I_consume_a_message(self): + self.result = [] + consumer = self.loop.run_until_complete( + self.queue.consume(self.result.append, exclusive=True)) + for retry in range(10): + test_utils.run_briefly(self.loop) + if self.result: + break + consumer.cancel() + + def it_should_return_my_message(self): + assert self.result, "Message not consumed" + assert self.result[0].body == b"Test message" + + def cleanup_loop(self): + @asyncio.coroutine + def tear_down(): + yield from self.channel.close() + yield from self.connection.close() + self.loop.run_until_complete(tear_down()) + asyncio.set_event_loop(self.loop) From e9aa720367a20a671de144f216789f4c05f280af Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 21 Sep 2015 22:36:47 +0100 Subject: [PATCH 064/118] Some tests were passing when they shouldn't. I added some code to the base test to throw unhandled exceptions, into the test framework. Then I fixed the broken tests in a rather hacky fashion. --- src/asynqp/channel.py | 8 +++++--- src/asynqp/connection.py | 6 ++++-- src/asynqp/routing.py | 3 ++- test/base_contexts.py | 16 +++++++++++++--- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 2ece4d4..b9c0658 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -98,6 +98,7 @@ def close(self): This method is a :ref:`coroutine `. """ + self._closing.set_result(True) self.sender.send_Close(0, 'Channel closed by application', 0, 0) yield from self.synchroniser.await(spec.ChannelCloseOK) # don't call self.reader.ready - stop reading frames from the q @@ -166,15 +167,16 @@ def open(self): consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - handler = ChannelActor(consumers, synchroniser, sender, loop=self.loop) - reader, writer = routing.create_reader_and_writer(handler, loop=self.loop) - handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) + actor = ChannelActor(consumers, synchroniser, sender, loop=self.loop) + reader, writer = routing.create_reader_and_writer(actor, loop=self.loop) + actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) queue_factory = queue.QueueFactory( sender, synchroniser, reader, consumers, loop=self.loop) channel = Channel( channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader, loop=self.loop) + channel._closing = actor.closing self.dispatcher.add_writer(channel_id, writer) try: diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 73367d7..fee0dc9 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -60,6 +60,7 @@ def close(self): This method is a :ref:`coroutine `. """ + self._closing.set_result(True) self.sender.send_Close(0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) self.closed.set_result(True) @@ -71,9 +72,10 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): sender = ConnectionMethodSender(protocol) connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) - handler = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) + actor = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) + connection._closing = actor.closing # bit ugly - reader, writer = routing.create_reader_and_writer(handler, loop=loop) + reader, writer = routing.create_reader_and_writer(actor, loop=loop) try: dispatcher.add_writer(0, writer) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index c207901..2bfe777 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -45,7 +45,8 @@ def __init__(self, synchroniser, sender, *, loop): self.closing = asyncio.Future(loop=self._loop) def handle(self, frame): - if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): + close_methods = (spec.ConnectionClose, spec.ConnectionCloseOK, spec.ChannelClose, spec.ChannelCloseOK) + if self.closing.done() and not isinstance(frame.payload, close_methods): return try: meth = getattr(self, 'handle_' + type(frame).__name__) diff --git a/test/base_contexts.py b/test/base_contexts.py index c2bfb8d..32bf71d 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -11,14 +11,24 @@ class LoopContext: def given_an_event_loop(self): + self.exceptions = [] self.loop = asyncio.get_event_loop() + self.loop.set_debug(True) + self.loop.set_exception_handler(self.exception_handler) asynqp.routing._TEST = True - def tick(self): - test_utils.run_briefly(self.loop) - def cleanup_test_hack(self): + self.loop.set_debug(False) + self.loop.set_exception_handler(None) asynqp.routing._TEST = False + if self.exceptions: + raise self.exceptions[0] + + def exception_handler(self, loop, context): + self.exceptions.append(context['exception']) + + def tick(self): + test_utils.run_briefly(self.loop) def async_partial(self, coro): """ From 72d103ff8e5b3ab7339c89aba67b9f2cea351f97 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 21 Sep 2015 23:31:25 +0100 Subject: [PATCH 065/118] Fix #45: HeartbeatMonitor now controls two tasks, not a chain of callbacks --- src/asynqp/protocol.py | 50 ++++++++++--------- test/base_contexts.py | 5 -- test/heartbeat_tests.py | 106 +++++++++++++++------------------------- 3 files changed, 67 insertions(+), 94 deletions(-) diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 9a082c1..841cd73 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -10,7 +10,7 @@ def __init__(self, dispatcher, loop): self.dispatcher = dispatcher self.partial_frame = b'' self.frame_reader = FrameReader() - self.heartbeat_monitor = HeartbeatMonitor(self, loop, 0) + self.heartbeat_monitor = HeartbeatMonitor(self, loop) def connection_made(self, transport): self.transport = transport @@ -83,32 +83,38 @@ def read_frame(self, data): class HeartbeatMonitor(object): - def __init__(self, protocol, loop, heartbeat_interval): + def __init__(self, protocol, loop): self.protocol = protocol self.loop = loop - self.heartbeat_interval = heartbeat_interval - self.heartbeat_timeout_callback = None + self.send_hb_task = None + self.monitor_task = None def start(self, interval): - if interval > 0: - self.heartbeat_interval = interval - self.send_heartbeat() - self.monitor_heartbeat() - - def send_heartbeat(self): - if self.heartbeat_interval > 0: + if interval <= 0: + return + self.send_hb_task = asyncio.async(self.send_heartbeat(interval), loop=self.loop) + self.monitor_task = asyncio.async(self.monitor_heartbeat(interval), loop=self.loop) + + def stop(self): + if self.send_hb_task is not None: + self.send_hb_task.cancel() + if self.monitor_task is not None: + self.monitor_task.cancel() + + @asyncio.coroutine + def send_heartbeat(self, interval): + while True: self.protocol.send_frame(frames.HeartbeatFrame()) - self.loop.call_later(self.heartbeat_interval, self.send_heartbeat) + yield from asyncio.sleep(interval) - def monitor_heartbeat(self): - if self.heartbeat_interval > 0: - self.heartbeat_timeout_callback = self.loop.call_later(self.heartbeat_interval * 2, self.heartbeat_timed_out) + @asyncio.coroutine + def monitor_heartbeat(self, interval): + while True: + self.is_alive = False + yield from asyncio.sleep(interval * 2) + if not self.is_alive: + self.protocol.send_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + self.protocol.connection_lost(ConnectionLostError('Heartbeat timed out')) def heartbeat_received(self): - if self.heartbeat_timeout_callback is not None: - self.heartbeat_timeout_callback.cancel() - self.monitor_heartbeat() - - def heartbeat_timed_out(self): - self.protocol.send_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - self.protocol.connection_lost(ConnectionLostError) + self.is_alive = True diff --git a/test/base_contexts.py b/test/base_contexts.py index 32bf71d..540609c 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -119,11 +119,6 @@ def given_a_consumer(self): self.consumer = task.result() -class MockLoopContext(LoopContext): - def given_an_event_loop(self): - self.loop = mock.Mock(spec=asyncio.AbstractEventLoop) - - class ProtocolContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index 59631ae..611cb54 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -1,84 +1,56 @@ -from unittest import mock +import asyncio from asynqp import spec -from asynqp import frames -from asynqp import protocol from asynqp.exceptions import ConnectionLostError -from .base_contexts import ProtocolContext, MockLoopContext +from asynqp.frames import HeartbeatFrame +from .base_contexts import MockServerContext -class WhenStartingTheHeartbeat(ProtocolContext, MockLoopContext): - def when_I_start_the_heartbeat(self): - self.protocol.start_heartbeat(5) +class WhenServerWaitsForHeartbeat(MockServerContext): + def when_heartbeating_starts(self): + self.protocol.start_heartbeat(0.01) + self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False + self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False + self.loop.run_until_complete(asyncio.sleep(0.015)) - def it_should_set_up_heartbeat_and_timeout_callbacks(self): - assert self.loop.call_later.call_args_list == [ - mock.call(5, self.protocol.heartbeat_monitor.send_heartbeat), - mock.call(10, self.protocol.heartbeat_monitor.heartbeat_timed_out) - ] + def it_should_send_the_heartbeat(self): + self.server.should_have_received_frame(HeartbeatFrame()) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() -class WhenHeartbeatIsDisabled(ProtocolContext, MockLoopContext): - def given_the_server_does_not_want_a_heartbeat(self): - self.heartbeat_interval = 0 - def when_I_start_the_heartbeat(self): - self.protocol.start_heartbeat(self.heartbeat_interval) +class WhenServerRespondsToHeartbeat(MockServerContext): + def given_i_started_heartbeating(self): + self.protocol.start_heartbeat(0.01) + self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False + self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False + self.loop.run_until_complete(asyncio.sleep(0.015)) - def it_should_not_set_up_callbacks(self): - assert not self.loop.call_later.called + def when_the_server_replies(self): + self.server.send_frame(HeartbeatFrame()) + self.loop.run_until_complete(asyncio.sleep(0.005)) + def it_should_send_the_heartbeat(self): + self.server.should_have_received_frames([HeartbeatFrame(), HeartbeatFrame()]) -class WhenItIsTimeToHeartbeat(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() - def when_the_event_loop_comes_a_knockin(self): - self.heartbeat_monitor.send_heartbeat() - def it_should_send_a_heartbeat_frame(self): - self.protocol.send_frame.assert_called_once_with(frames.HeartbeatFrame()) +class WhenServerDoesNotRespondToHeartbeat(MockServerContext): + def given_i_started_heartbeating(self): + self.protocol.start_heartbeat(0.01) + self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False + self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False - def it_should_set_up_the_next_heartbeat(self): - self.loop.call_later.assert_called_once_with(5, self.heartbeat_monitor.send_heartbeat) + def when_the_server_dies(self): + self.loop.run_until_complete(asyncio.sleep(0.021)) + def it_should_close_the_connection(self): + self.server.should_have_received_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) -class WhenResettingTheHeartbeatTimeout(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) - self.heartbeat_monitor.monitor_heartbeat() - self.loop.reset_mock() + def it_should_throw(self): + assert isinstance(self.protocol.heartbeat_monitor.monitor_task.exception(), ConnectionLostError) - def because_the_timeout_gets_reset(self): - self.heartbeat_monitor.heartbeat_received() - - def it_should_cancel_the_close_callback(self): - self.loop.call_later.return_value.cancel.assert_called_once_with() - - def it_should_set_up_another_close_callback(self): - self.loop.call_later.assert_called_once_with(10, self.heartbeat_monitor.heartbeat_timed_out) - - -class WhenTheHeartbeatTimesOut(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) - - def when_the_heartbeat_times_out(self): - self.heartbeat_monitor.heartbeat_timed_out() - - def it_should_send_connection_close(self): - self.protocol.send_method.assert_called_once_with(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - - -class WhenTheHeartbeatTimesOutCallProtocolConnectionLost(MockLoopContext): - def given_a_hearbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) - - def when_the_heartbeat_times_out(self): - self.heartbeat_monitor.heartbeat_timed_out() - - def it_should_call_protocol_lost_connection(self): - self.protocol.connection_lost.assert_called_once_with(ConnectionLostError) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() From 3b93de3468efd2d73fafd42e4f8707d23ff1fc5d Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 21 Sep 2015 23:34:02 +0100 Subject: [PATCH 066/118] #45 delete obsolete tests --- test/integration_tests.py | 49 --------------------------------------- 1 file changed, 49 deletions(-) diff --git a/test/integration_tests.py b/test/integration_tests.py index c55c87c..1a9cbaa 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -226,55 +226,6 @@ def it_should_not_throw(self): assert self.exception is None -# class WhenAConnectionIsClosed: -# def given_an_exception_handler_and_connection(self): -# self.loop = asyncio.get_event_loop() -# self.connection_closed_error_raised = False -# self.loop.set_exception_handler(self.exception_handler) -# self.connection = self.loop.run_until_complete(asynqp.connect()) - -# def exception_handler(self, loop, context): -# self.exception = context.get('exception') -# self.loop.default_exception_handler(context) - -# def when_the_connection_is_closed(self): -# self.loop.run_until_complete(self.connection.close()) - -# def it_should_raise_a_connection_closed_error(self): -# assert self.exception is None - -# def cleanup(self): -# self.loop.set_exception_handler(testing_exception_handler) - - -class WhenAConnectionIsLost: - def given_an_exception_handler_and_connection(self): - self.loop = asyncio.get_event_loop() - self.connection_lost_error_raised = False - self.loop.set_exception_handler(self.exception_handler) - self.connection = self.loop.run_until_complete(asynqp.connect()) - - def exception_handler(self, loop, context): - exception = context.get('exception') - if type(exception) is asynqp.exceptions.ConnectionLostError: - self.connection_lost_error_raised = True - self.loop.stop() - else: - self.loop.default_exception_handler(context) - - def when_the_heartbeat_times_out(self): - self.loop.call_soon(self.connection - .protocol - .heartbeat_monitor.heartbeat_timed_out) - self.loop.run_forever() - - def it_should_raise_a_connection_closed_error(self): - assert self.connection_lost_error_raised is True - - def cleanup(self): - self.loop.set_exception_handler(testing_exception_handler) - - class WhenAConnectionIsClosedCloseConnection: def given_a_connection(self): self.loop = asyncio.get_event_loop() From 189e11d64be18fa20f9a95558e6f97686747b7c1 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Mon, 21 Sep 2015 23:46:37 +0100 Subject: [PATCH 067/118] keep flake8 happy --- test/integration_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration_tests.py b/test/integration_tests.py index 1a9cbaa..f7175e0 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -3,7 +3,6 @@ import socket import contexts from asyncio import test_utils -from .util import testing_exception_handler class ConnectionContext: From 15307f9427151744fc7fc0ef3750c1b3d3f34ff4 Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 22 Sep 2015 10:56:38 +0300 Subject: [PATCH 068/118] Fixed great number of tracebacks in tests for not closed heartbeat. Fixed heartbeat logic to be more accurate on interval. --- src/asynqp/connection.py | 5 +++++ src/asynqp/protocol.py | 40 +++++++++++++++++++++++++++++++++------- test/base_contexts.py | 5 +++++ test/heartbeat_tests.py | 16 ++++++++-------- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index fee0dc9..ced9692 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -64,6 +64,11 @@ def close(self): self.sender.send_Close(0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) self.closed.set_result(True) + # Close heartbeat + # TODO: We really need a better solution for finalization of parts + # in the library. + self.protocol.heartbeat_monitor.stop() + yield from self.protocol.heartbeat_monitor.wait_closed() @asyncio.coroutine diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 841cd73..4c274d6 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -88,6 +88,7 @@ def __init__(self, protocol, loop): self.loop = loop self.send_hb_task = None self.monitor_task = None + self._last_received = 0 def start(self, interval): if interval <= 0: @@ -101,20 +102,45 @@ def stop(self): if self.monitor_task is not None: self.monitor_task.cancel() + @asyncio.coroutine + def wait_closed(self): + if self.send_hb_task is not None: + try: + yield from self.send_hb_task + except asyncio.CancelledError: + pass + if self.monitor_task is not None: + try: + yield from self.monitor_task + except asyncio.CancelledError: + pass + @asyncio.coroutine def send_heartbeat(self, interval): while True: self.protocol.send_frame(frames.HeartbeatFrame()) - yield from asyncio.sleep(interval) + yield from asyncio.sleep(interval, loop=self.loop) @asyncio.coroutine def monitor_heartbeat(self, interval): + self._last_received = self.loop.time() + no_beat_for = 0 while True: - self.is_alive = False - yield from asyncio.sleep(interval * 2) - if not self.is_alive: - self.protocol.send_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - self.protocol.connection_lost(ConnectionLostError('Heartbeat timed out')) + # We use interval roundtrip so 2x + yield from asyncio.sleep( + interval * 2 - no_beat_for, loop=self.loop) + + no_beat_for = self.loop.time() - self._last_received + if no_beat_for > interval * 2: + self.protocol.send_method( + 0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + # It's raised for backward compatibility + try: + self.protocol.connection_lost( + ConnectionLostError('Heartbeat timed out')) + except ConnectionLostError: + pass + break def heartbeat_received(self): - self.is_alive = True + self._last_received = self.loop.time() diff --git a/test/base_contexts.py b/test/base_contexts.py index 540609c..ad967a3 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -67,6 +67,11 @@ def given_an_open_connection(self): self.connection = task.result() + def cleanup_connection(self): + self.connection.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + self.connection.protocol.heartbeat_monitor.wait_closed()) + class OpenChannelContext(OpenConnectionContext): def given_an_open_channel(self): diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index 611cb54..c9fceca 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -8,8 +8,6 @@ class WhenServerWaitsForHeartbeat(MockServerContext): def when_heartbeating_starts(self): self.protocol.start_heartbeat(0.01) - self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False - self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False self.loop.run_until_complete(asyncio.sleep(0.015)) def it_should_send_the_heartbeat(self): @@ -17,13 +15,13 @@ def it_should_send_the_heartbeat(self): def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + self.protocol.heartbeat_monitor.wait_closed()) class WhenServerRespondsToHeartbeat(MockServerContext): def given_i_started_heartbeating(self): self.protocol.start_heartbeat(0.01) - self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False - self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False self.loop.run_until_complete(asyncio.sleep(0.015)) def when_the_server_replies(self): @@ -35,13 +33,13 @@ def it_should_send_the_heartbeat(self): def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + self.protocol.heartbeat_monitor.wait_closed()) class WhenServerDoesNotRespondToHeartbeat(MockServerContext): def given_i_started_heartbeating(self): self.protocol.start_heartbeat(0.01) - self.protocol.heartbeat_monitor.send_hb_task._log_destroy_pending = False - self.protocol.heartbeat_monitor.monitor_task._log_destroy_pending = False def when_the_server_dies(self): self.loop.run_until_complete(asyncio.sleep(0.021)) @@ -49,8 +47,10 @@ def when_the_server_dies(self): def it_should_close_the_connection(self): self.server.should_have_received_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - def it_should_throw(self): - assert isinstance(self.protocol.heartbeat_monitor.monitor_task.exception(), ConnectionLostError) + # def it_should_throw(self): + # assert isinstance(self.protocol.heartbeat_monitor.monitor_task.exception(), ConnectionLostError) def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + self.protocol.heartbeat_monitor.wait_closed()) From bdf94026004e7f8123bdfe1011c23a9c8ded3714 Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 22 Sep 2015 11:07:28 +0300 Subject: [PATCH 069/118] Fix flake8 --- test/heartbeat_tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index c9fceca..2cdc3df 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -1,6 +1,5 @@ import asyncio from asynqp import spec -from asynqp.exceptions import ConnectionLostError from asynqp.frames import HeartbeatFrame from .base_contexts import MockServerContext @@ -47,9 +46,6 @@ def when_the_server_dies(self): def it_should_close_the_connection(self): self.server.should_have_received_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - # def it_should_throw(self): - # assert isinstance(self.protocol.heartbeat_monitor.monitor_task.exception(), ConnectionLostError) - def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() self.loop.run_until_complete( From 2ffa80267dc10ec5b148273ca1382c6031994073 Mon Sep 17 00:00:00 2001 From: Taras Date: Wed, 23 Sep 2015 10:58:08 +0300 Subject: [PATCH 070/118] Minor fix to heartbeat --- src/asynqp/connection.py | 2 +- test/heartbeat_tests.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index ced9692..02bd070 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -63,12 +63,12 @@ def close(self): self._closing.set_result(True) self.sender.send_Close(0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) - self.closed.set_result(True) # Close heartbeat # TODO: We really need a better solution for finalization of parts # in the library. self.protocol.heartbeat_monitor.stop() yield from self.protocol.heartbeat_monitor.wait_closed() + self.closed.set_result(True) @asyncio.coroutine diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index 2cdc3df..9fe7574 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -15,7 +15,8 @@ def it_should_send_the_heartbeat(self): def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() self.loop.run_until_complete( - self.protocol.heartbeat_monitor.wait_closed()) + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) class WhenServerRespondsToHeartbeat(MockServerContext): @@ -33,7 +34,8 @@ def it_should_send_the_heartbeat(self): def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() self.loop.run_until_complete( - self.protocol.heartbeat_monitor.wait_closed()) + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) class WhenServerDoesNotRespondToHeartbeat(MockServerContext): @@ -49,4 +51,5 @@ def it_should_close_the_connection(self): def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() self.loop.run_until_complete( - self.protocol.heartbeat_monitor.wait_closed()) + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) From 29b399003e5520116c0c1dd58eded918e3f46476 Mon Sep 17 00:00:00 2001 From: Taras Date: Wed, 23 Sep 2015 11:06:10 +0300 Subject: [PATCH 071/118] Fixed connection test to wait a bit before checking --- test/connection_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/connection_tests.py b/test/connection_tests.py index 2de8300..a6f5cf3 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -65,6 +65,7 @@ def given_a_connection_that_I_closed(self): def when_connection_close_ok_arrives(self): self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() def it_should_close_the_transport(self): assert self.transport.closed From 02d9ddff52e3d8c1ebd8be273f633c2a56b406f7 Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 12 Sep 2015 17:45:57 +0300 Subject: [PATCH 072/118] Fix connection close after graceful close from server --- src/asynqp/connection.py | 19 ++++++++++--------- test/connection_tests.py | 4 ++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 02bd070..dff9bae 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -60,15 +60,16 @@ def close(self): This method is a :ref:`coroutine `. """ - self._closing.set_result(True) - self.sender.send_Close(0, 'Connection closed by application', 0, 0) - yield from self.synchroniser.await(spec.ConnectionCloseOK) - # Close heartbeat - # TODO: We really need a better solution for finalization of parts - # in the library. - self.protocol.heartbeat_monitor.stop() - yield from self.protocol.heartbeat_monitor.wait_closed() - self.closed.set_result(True) + if not self.closed.done(): + self._closing.set_result(True) + self.sender.send_Close(0, 'Connection closed by application', 0, 0) + yield from self.synchroniser.await(spec.ConnectionCloseOK) + # Close heartbeat + # TODO: We really need a better solution for finalization of parts + # in the library. + self.protocol.heartbeat_monitor.stop() + yield from self.protocol.heartbeat_monitor.wait_closed() + self.closed.set_result(True) @asyncio.coroutine diff --git a/test/connection_tests.py b/test/connection_tests.py index a6f5cf3..b16077d 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -48,6 +48,10 @@ def it_should_send_close_ok(self): def it_should_set_the_future(self): assert self.connection.closed.done() + def it_should_not_block_clonnection_close(self): + self.loop.run_until_complete( + asyncio.wait_for(self.connection.close(), 0.2)) + class WhenTheApplicationClosesTheConnection(OpenConnectionContext): def when_I_close_the_connection(self): From 443836782f44470b2577c5cadb435e9c76cf9331 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 21 Sep 2015 21:36:34 +0300 Subject: [PATCH 073/118] Log warning if called `close` on already closed connection --- src/asynqp/connection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index dff9bae..8063502 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -1,9 +1,12 @@ import asyncio +import logging import sys from . import channel from . import spec from . import routing +log = logging.getLogger(__name__) + class Connection(object): """ @@ -70,6 +73,8 @@ def close(self): self.protocol.heartbeat_monitor.stop() yield from self.protocol.heartbeat_monitor.wait_closed() self.closed.set_result(True) + else: + log.warn("Called `close` on already closed connection...") @asyncio.coroutine From ad14f65a680f0607e53413c8b44b5b9538cd3972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=BE=D1=80=D0=B5=D0=BD=D0=B1=D0=B5=D1=80=D0=B3=20?= =?UTF-8?q?=D0=9C=D0=B0=D1=80=D0=BA?= Date: Sat, 19 Sep 2015 10:33:58 +0500 Subject: [PATCH 074/118] Fix #40 set TCP_NODELAY by default --- src/asynqp/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index 027fa55..dddbf08 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,3 +1,4 @@ +import socket import asyncio from .exceptions import * # noqa from .message import Message, IncomingMessage @@ -40,6 +41,8 @@ def connect(host='localhost', Further keyword arguments are passed on to :meth:`loop.create_connection() `. + This function will set TCP_NODELAY on TCP and TCP6 sockets either on supplied ``sock`` or created one. + :return: the :class:`Connection` object. """ from .protocol import AMQP @@ -57,6 +60,15 @@ def connect(host='localhost', dispatcher = Dispatcher() transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) + # RPC-like applications require TCP_NODELAY in order to acheive + # minimal response time. Actually, this library send data in one + # big chunk and so this will not affect TCP-performance. + sk = transport.get_extra_info('socket') + # 1. Unfortunatelly we cannot check socket type (sk.type == socket.SOCK_STREAM). https://bugs.python.org/issue21327 + # 2. Proto remains zero, if not specified at creation of socket + if (sk.family in (socket.AF_INET, socket.AF_INET6)) and (sk.proto in (0, socket.IPPROTO_TCP)): + sk.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + connection = yield from open_connection(loop, transport, protocol, dispatcher, {'username': username, 'password': password, 'virtual_host': virtual_host}) return connection From baa63427affa47e725dde1193d17248471f2437e Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 4 Oct 2015 15:28:16 +0300 Subject: [PATCH 075/118] Refactored reader to not use async calls --- src/asynqp/routing.py | 49 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 2bfe777..521bfb6 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -107,9 +107,8 @@ def killall(self, exc): def create_reader_and_writer(handler, *, loop): - q = asyncio.Queue(loop=loop) - reader = QueueReader(handler, q, loop=loop) - writer = QueueWriter(q) + reader = QueueReader(handler, loop=loop) + writer = QueueWriter(reader) return reader, writer @@ -117,38 +116,38 @@ def create_reader_and_writer(handler, *, loop): # When the frame does arrive, dispatch it to the handler and do nothing # until someone calls ready() again. class QueueReader(object): - def __init__(self, handler, q, *, loop): - self._loop = loop + def __init__(self, handler, *, loop): self.handler = handler - self.q = q self.is_waiting = False + self.pending_frames = collections.deque() + self._loop = loop def ready(self): assert not self.is_waiting, "ready() got called while waiting for a frame to be read" - self.is_waiting = True - - # XXX: Refactor this. There should be only 1 async task per QueueReader - # It will read frames in a `while True:` loop and will be canceled - # when connection is closed. - - t = asyncio.async(self._read_next(), loop=self._loop) - if _TEST: # this feels hacky to me - t._log_destroy_pending = False - - @asyncio.coroutine - def _read_next(self): - assert self.is_waiting, "a frame got read without ready() having been called" - frame = yield from self.q.get() - self.is_waiting = False - self.handler.handle(frame) + if self.pending_frames: + frame = self.pending_frames.popleft() + # We will call it in another tick just to be more strict about the + # sequence of frames + self._loop.call_soon(self.handler.handle, frame) + else: + self.is_waiting = True + + def feed(self, frame): + if self.is_waiting: + self.is_waiting = False + # We will call it in another tick just to be more strict about the + # sequence of frames + self._loop.call_soon(self.handler.handle, frame) + else: + self.pending_frames.append(frame) class QueueWriter(object): - def __init__(self, q): - self.q = q + def __init__(self, reader): + self.reader = reader def enqueue(self, frame): - self.q.put_nowait(frame) + self.reader.feed(frame) class OrderedManyToManyMap(object): From 274cdbce46ae031d2a3c9205f74cfdbf0fd620ee Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 4 Oct 2015 16:14:48 +0300 Subject: [PATCH 076/118] Refactor message_receiver to not use async tasks on message consuming --- src/asynqp/channel.py | 86 +++++++++++++++++++++++-------------------- src/asynqp/queue.py | 9 ++--- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index b9c0658..a5835f0 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -1,5 +1,7 @@ import asyncio import re +from functools import partial + from . import frames from . import spec from . import queue @@ -232,11 +234,8 @@ def handle_QueueDeleteOK(self, frame): self.synchroniser.notify(spec.QueueDeleteOK) def handle_BasicGetEmpty(self, frame): - self.synchroniser.notify(spec.BasicGetEmpty, False) - - def handle_BasicGetOK(self, frame): - assert self.message_receiver is not None, "message_receiver not set" - asyncio.async(self.message_receiver.receive_getOK(frame), loop=self._loop) + # Send result=None to notify Empty message + self.synchroniser.notify(spec.BasicGetEmpty, None) def handle_BasicConsumeOK(self, frame): self.synchroniser.notify(spec.BasicConsumeOK, frame.payload.consumer_tag) @@ -244,18 +243,6 @@ def handle_BasicConsumeOK(self, frame): def handle_BasicCancelOK(self, frame): self.synchroniser.notify(spec.BasicCancelOK) - def handle_BasicDeliver(self, frame): - assert self.message_receiver is not None, "message_receiver not set" - asyncio.async(self.message_receiver.receive_deliver(frame), loop=self._loop) - - def handle_ContentHeaderFrame(self, frame): - assert self.message_receiver is not None, "message_receiver not set" - asyncio.async(self.message_receiver.receive_header(frame), loop=self._loop) - - def handle_ContentBodyFrame(self, frame): - assert self.message_receiver is not None, "message_receiver not set" - asyncio.async(self.message_receiver.receive_body(frame), loop=self._loop) - def handle_ChannelClose(self, frame): self.sender.send_CloseOK() exc = exceptions._get_exception_type(frame.payload.reply_code) @@ -267,9 +254,28 @@ def handle_ChannelCloseOK(self, frame): def handle_BasicQosOK(self, frame): self.synchroniser.notify(spec.BasicQosOK) + # Message receiving hanlers + + def handle_BasicGetOK(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + # Syncronizer will be notified after full msg is consumed + self.message_receiver.receive_getOK(frame) + + def handle_BasicDeliver(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_deliver(frame) + + def handle_ContentHeaderFrame(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_header(frame) + + def handle_ContentBodyFrame(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_body(frame) + def handle_BasicReturn(self, frame): assert self.message_receiver is not None, "message_receiver not set" - asyncio.async(self.message_receiver.receive_return(frame), loop=self._loop) + self.message_receiver.receive_return(frame) class MessageReceiver(object): @@ -279,10 +285,9 @@ def __init__(self, synchroniser, sender, consumers, reader): self.consumers = consumers self.reader = reader self.message_builder = None + self.message_callback = None - @asyncio.coroutine def receive_getOK(self, frame): - self.synchroniser.notify(spec.BasicGetOK, True) payload = frame.payload self.message_builder = message.MessageBuilder( self.sender, @@ -291,9 +296,11 @@ def receive_getOK(self, frame): payload.exchange, payload.routing_key ) + # Send result=(msg, tag) when full message is returned + self.message_callback = partial( + self.synchroniser.notify, spec.BasicGetOK) self.reader.ready() - @asyncio.coroutine def receive_deliver(self, frame): payload = frame.payload self.message_builder = message.MessageBuilder( @@ -304,11 +311,14 @@ def receive_deliver(self, frame): payload.routing_key, payload.consumer_tag ) - self.reader.ready() - yield from self.async_receive() + # Deliver the message to consumers when done + def callback(tag_msg): + self.consumers.deliver(*tag_msg) + self.reader.ready() + self.message_callback = callback + self.reader.ready() - @asyncio.coroutine def receive_return(self, frame): payload = frame.payload self.message_builder = message.MessageBuilder( @@ -319,36 +329,32 @@ def receive_return(self, frame): payload.routing_key, BasicReturnConsumer.tag ) - self.reader.ready() - - yield from self.async_receive() - @asyncio.coroutine - def async_receive(self): - yield from self.synchroniser.await(frames.ContentHeaderFrame) - tag, msg = yield from self.synchroniser.await(frames.ContentBodyFrame) - - self.consumers.deliver(tag, msg) + # Deliver the message to BasicReturnConsumer when done + def callback(tag_msg): + self.consumers.deliver(*tag_msg) + self.reader.ready() + self.message_callback = callback self.reader.ready() - @asyncio.coroutine def receive_header(self, frame): - self.synchroniser.notify(frames.ContentHeaderFrame) + assert self.message_builder is not None, "Reveived unexpected header" self.message_builder.set_header(frame.payload) self.reader.ready() - @asyncio.coroutine def receive_body(self, frame): + assert self.message_builder is not None, "Reveived unexpected body" self.message_builder.add_body_chunk(frame.payload) if self.message_builder.done(): msg = self.message_builder.build() tag = self.message_builder.consumer_tag - self.synchroniser.notify(frames.ContentBodyFrame, (tag, msg)) + self.message_callback((tag, msg)) self.message_builder = None - # don't call self.reader.ready() if the message is all here - - # get() or async_receive() will call - # it when they have finished processing the completed msg + self.message_callback = None + # Dont call ready() if full message arrive. It's the original + # caller's responsibility return + # If message is not done yet we still need more frames. Wait for them self.reader.ready() diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index f47352d..07d9608 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -1,7 +1,7 @@ import asyncio import re from operator import delitem -from . import spec, frames +from . import spec from .exceptions import Deleted @@ -130,11 +130,10 @@ def get(self, *, no_ack=False): raise Deleted("Queue {} was deleted".format(self.name)) self.sender.send_BasicGet(self.name, no_ack) - has_message = yield from self.synchroniser.await(spec.BasicGetOK, spec.BasicGetEmpty) + tag_msg = yield from self.synchroniser.await(spec.BasicGetOK, spec.BasicGetEmpty) - if has_message: - yield from self.synchroniser.await(frames.ContentHeaderFrame) - consumer_tag, msg = yield from self.synchroniser.await(frames.ContentBodyFrame) + if tag_msg is not None: + consumer_tag, msg = tag_msg assert consumer_tag is None else: msg = None From 33afaa5ac0e276c8817cfc0c7d3a4f2a29e970d9 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 5 Oct 2015 20:10:02 +0300 Subject: [PATCH 077/118] Removed `callback` from message_receiver. Let `receive_body` decide how to deliver. --- src/asynqp/channel.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index a5835f0..6ba8c4c 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -285,7 +285,7 @@ def __init__(self, synchroniser, sender, consumers, reader): self.consumers = consumers self.reader = reader self.message_builder = None - self.message_callback = None + self.is_getok_message = None def receive_getOK(self, frame): payload = frame.payload @@ -296,9 +296,8 @@ def receive_getOK(self, frame): payload.exchange, payload.routing_key ) - # Send result=(msg, tag) when full message is returned - self.message_callback = partial( - self.synchroniser.notify, spec.BasicGetOK) + # Send message to synchroniser when done + self.is_getok_message = True self.reader.ready() def receive_deliver(self, frame): @@ -312,11 +311,8 @@ def receive_deliver(self, frame): payload.consumer_tag ) - # Deliver the message to consumers when done - def callback(tag_msg): - self.consumers.deliver(*tag_msg) - self.reader.ready() - self.message_callback = callback + # Delivers message to consumers when done + self.is_getok_message = False self.reader.ready() def receive_return(self, frame): @@ -330,11 +326,8 @@ def receive_return(self, frame): BasicReturnConsumer.tag ) - # Deliver the message to BasicReturnConsumer when done - def callback(tag_msg): - self.consumers.deliver(*tag_msg) - self.reader.ready() - self.message_callback = callback + # Delivers message to BasicReturnConsumer when done + self.is_getok_message = False self.reader.ready() def receive_header(self, frame): @@ -348,11 +341,15 @@ def receive_body(self, frame): if self.message_builder.done(): msg = self.message_builder.build() tag = self.message_builder.consumer_tag - self.message_callback((tag, msg)) + if self.is_getok_message: + self.synchroniser.notify(spec.BasicGetOK, (tag, msg)) + # Dont call ready() if message arrive after GetOk. It's the + # ``Queue.get`` method's responsibility + else: + self.consumers.deliver(tag, msg) + self.reader.ready() + self.message_builder = None - self.message_callback = None - # Dont call ready() if full message arrive. It's the original - # caller's responsibility return # If message is not done yet we still need more frames. Wait for them self.reader.ready() From 4c24984b33ec0521ffa4ec2dfd3746ec0bce7130 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 5 Oct 2015 20:32:56 +0300 Subject: [PATCH 078/118] Removed QueueWriter and passing Reader instead. Renamed QueueReader to QueuedReader. --- src/asynqp/channel.py | 6 +++--- src/asynqp/connection.py | 7 +++---- src/asynqp/routing.py | 34 ++++++++++------------------------ 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index b9c0658..76eb1b3 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -168,7 +168,7 @@ def open(self): consumers.add_consumer(basic_return_consumer) actor = ChannelActor(consumers, synchroniser, sender, loop=self.loop) - reader, writer = routing.create_reader_and_writer(actor, loop=self.loop) + reader = routing.QueuedReader(actor, loop=self.loop) actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) queue_factory = queue.QueueFactory( @@ -178,7 +178,7 @@ def open(self): queue_factory, reader, loop=self.loop) channel._closing = actor.closing - self.dispatcher.add_writer(channel_id, writer) + self.dispatcher.add_handler(channel_id, reader.feed) try: sender.send_ChannelOpen() reader.ready() @@ -191,7 +191,7 @@ def open(self): # If we leave self.next_channel_id incremented, the worst # that happens is we end up with non-sequential channel numbers. # Small price to pay to keep this method re-entrant. - self.dispatcher.remove_writer(channel_id) + self.dispatcher.remove_handler(channel_id) raise reader.ready() diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 8063502..62331f4 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -85,11 +85,10 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) actor = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) connection._closing = actor.closing # bit ugly - - reader, writer = routing.create_reader_and_writer(actor, loop=loop) + reader = routing.QueuedReader(actor, loop=loop) try: - dispatcher.add_writer(0, writer) + dispatcher.add_handler(0, reader.feed) protocol.send_protocol_header() reader.ready() @@ -117,7 +116,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): yield from synchroniser.await(spec.ConnectionOpenOK) reader.ready() except: - dispatcher.remove_writer(0) + dispatcher.remove_handler(0) raise return connection diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 521bfb6..8518572 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -9,23 +9,23 @@ class Dispatcher(object): def __init__(self): - self.queue_writers = {} + self.handlers = {} - def add_writer(self, channel_id, writer): - self.queue_writers[channel_id] = writer + def add_handler(self, channel_id, handler): + self.handlers[channel_id] = handler - def remove_writer(self, channel_id): - del self.queue_writers[channel_id] + def remove_handler(self, channel_id): + del self.handlers[channel_id] def dispatch(self, frame): if isinstance(frame, frames.HeartbeatFrame): return - writer = self.queue_writers[frame.channel_id] - writer.enqueue(frame) + handler = self.handlers[frame.channel_id] + handler(frame) def dispatch_all(self, frame): - for writer in self.queue_writers.values(): - writer.enqueue(frame) + for handler in self.handlers.values(): + handler(frame) class Sender(object): @@ -106,16 +106,10 @@ def killall(self, exc): self._futures.remove_item(fut) -def create_reader_and_writer(handler, *, loop): - reader = QueueReader(handler, loop=loop) - writer = QueueWriter(reader) - return reader, writer - - # When ready() is called, wait for a frame to arrive on the queue. # When the frame does arrive, dispatch it to the handler and do nothing # until someone calls ready() again. -class QueueReader(object): +class QueuedReader(object): def __init__(self, handler, *, loop): self.handler = handler self.is_waiting = False @@ -142,14 +136,6 @@ def feed(self, frame): self.pending_frames.append(frame) -class QueueWriter(object): - def __init__(self, reader): - self.reader = reader - - def enqueue(self, frame): - self.reader.feed(frame) - - class OrderedManyToManyMap(object): def __init__(self): self._items = collections.defaultdict(OrderedSet) From b7f99c8e2ddf60d70af0a13ea26bc8305c7ba602 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 5 Oct 2015 20:39:25 +0300 Subject: [PATCH 079/118] Flake8 and spelling fix --- src/asynqp/channel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 6ba8c4c..19802b3 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -1,6 +1,5 @@ import asyncio import re -from functools import partial from . import frames from . import spec @@ -331,12 +330,12 @@ def receive_return(self, frame): self.reader.ready() def receive_header(self, frame): - assert self.message_builder is not None, "Reveived unexpected header" + assert self.message_builder is not None, "Received unexpected header" self.message_builder.set_header(frame.payload) self.reader.ready() def receive_body(self, frame): - assert self.message_builder is not None, "Reveived unexpected body" + assert self.message_builder is not None, "Received unexpected body" self.message_builder.add_body_chunk(frame.payload) if self.message_builder.done(): msg = self.message_builder.build() From 50ba10ac36216116a5be720b567d2ba91eaad692 Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 4 Oct 2015 15:58:39 +0300 Subject: [PATCH 080/118] Refactored Synchroniser to not use strage constructs based on OrderedDict --- src/asynqp/channel.py | 7 ++- src/asynqp/connection.py | 6 ++- src/asynqp/exceptions.py | 7 ++- src/asynqp/log.py | 3 ++ src/asynqp/protocol.py | 7 ++- src/asynqp/queue.py | 11 ++-- src/asynqp/routing.py | 106 +++++++++------------------------------ 7 files changed, 58 insertions(+), 89 deletions(-) create mode 100644 src/asynqp/log.py diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 2c2ac0b..fea0625 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -8,7 +8,7 @@ from . import exchange from . import message from . import routing -from .exceptions import UndeliverableMessage +from .exceptions import UndeliverableMessage, AlreadyClosed VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -101,7 +101,10 @@ def close(self): """ self._closing.set_result(True) self.sender.send_Close(0, 'Channel closed by application', 0, 0) - yield from self.synchroniser.await(spec.ChannelCloseOK) + try: + yield from self.synchroniser.await(spec.ChannelCloseOK) + except AlreadyClosed: + pass # don't call self.reader.ready - stop reading frames from the q @asyncio.coroutine diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 62331f4..7eea001 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -4,6 +4,7 @@ from . import channel from . import spec from . import routing +from .exceptions import AlreadyClosed log = logging.getLogger(__name__) @@ -66,7 +67,10 @@ def close(self): if not self.closed.done(): self._closing.set_result(True) self.sender.send_Close(0, 'Connection closed by application', 0, 0) - yield from self.synchroniser.await(spec.ConnectionCloseOK) + try: + yield from self.synchroniser.await(spec.ConnectionCloseOK) + except AlreadyClosed: + pass # Close heartbeat # TODO: We really need a better solution for finalization of parts # in the library. diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index b493309..8968466 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -11,7 +11,12 @@ __all__.extend(EXCEPTIONS.keys()) -class ConnectionLostError(ConnectionError): +class AlreadyClosed(Exception): + """ Raised when issuing commands on closed Channel/Connection. + """ + + +class ConnectionLostError(AlreadyClosed, ConnectionError): ''' Connection was closed unexpectedly ''' diff --git a/src/asynqp/log.py b/src/asynqp/log.py new file mode 100644 index 0000000..c34e86a --- /dev/null +++ b/src/asynqp/log.py @@ -0,0 +1,3 @@ +import logging + +log = logging.getLogger("asynqp") diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 4c274d6..3644411 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -46,7 +46,12 @@ def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) def connection_lost(self, exc): - self.dispatcher.dispatch_all(frames.PoisonPillFrame(exc)) + if exc is None: + poison_exc = ConnectionLostError( + 'The connection was unexpectedly lost') + else: + poison_exc = exc + self.dispatcher.dispatch_all(frames.PoisonPillFrame(poison_exc)) if exc is not None: raise ConnectionLostError('The connection was unexpectedly lost') from exc diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index 07d9608..fee353b 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -2,7 +2,7 @@ import re from operator import delitem from . import spec -from .exceptions import Deleted +from .exceptions import Deleted, AlreadyClosed VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -258,12 +258,17 @@ def cancel(self): This method is a :ref:`coroutine `. """ self.sender.send_BasicCancel(self.tag) - yield from self.synchroniser.await(spec.BasicCancelOK) + try: + yield from self.synchroniser.await(spec.BasicCancelOK) + except AlreadyClosed: + pass + else: + # No need to call ready if connection closed. + self.reader.ready() self.cancelled = True self.cancelled_future.set_result(self) if hasattr(self.callback, 'on_cancel'): self.callback.on_cancel() - self.reader.ready() class QueueFactory(object): diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 8518572..c03aa92 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -2,6 +2,7 @@ import collections from . import frames from . import spec +from .log import log _TEST = False @@ -56,54 +57,48 @@ def handle(self, frame): meth(frame) def handle_PoisonPillFrame(self, frame): - self.synchroniser.killall(ConnectionError) + self.synchroniser.killall(frame.exception) class Synchroniser(object): - _blocking_methods = set((spec.BasicCancelOK, # Consumer.cancel - spec.ChannelCloseOK, # Channel.close - spec.ConnectionCloseOK)) # Connection.close def __init__(self, *, loop): self._loop = loop - self._futures = OrderedManyToManyMap() - self.connection_closed = False + self._futures = collections.defaultdict(collections.deque) + self.connection_exc = None def await(self, *expected_methods): fut = asyncio.Future(loop=self._loop) - if self.connection_closed: - for method in expected_methods: - if method in self._blocking_methods and not fut.done(): - fut.set_result(None) - if not fut.done(): - fut.set_exception(ConnectionError) + if self.connection_exc is not None: + fut.set_exception(self.connection_exc) return fut - self._futures.add_item(expected_methods, fut) + for method in expected_methods: + self._futures[method].append((fut, expected_methods)) return fut def notify(self, method, result=None): - fut = self._futures.get_leftmost(method) - fut.set_result(result) - self._futures.remove_item(fut) + try: + fut, bound_methods = self._futures[method].popleft() + except IndexError: + # XXX: we can't just ignore this. + log.error("Got an unexpected method notification %s", method) + else: + # Cleanup futures, that were awaited together, like + # (spec.BasicGetOK, spec.BasicGetEmpty) + for other_method in bound_methods: + if other_method != method: + self._futures[other_method].remove((fut, bound_methods)) + fut.set_result(result) def killall(self, exc): - self.connection_closed = True - # Give a proper notification to methods which are waiting for closure - for method in self._blocking_methods: - while True: - try: - self.notify(method) - except StopIteration: - break - + self.connection_exc = exc # Set an exception for all others - for method in self._futures.keys(): - if method not in self._blocking_methods: - for fut in self._futures.get_all(method): - fut.set_exception(exc) - self._futures.remove_item(fut) + for method, futs in self._futures.items(): + for fut, _ in futs: + fut.set_exception(exc) + self._futures.clear() # When ready() is called, wait for a frame to arrive on the queue. @@ -134,54 +129,3 @@ def feed(self, frame): self._loop.call_soon(self.handler.handle, frame) else: self.pending_frames.append(frame) - - -class OrderedManyToManyMap(object): - def __init__(self): - self._items = collections.defaultdict(OrderedSet) - - def add_item(self, keys, item): - for key in keys: - self._items[key].add(item) - - def remove_item(self, item): - for ordered_set in self._items.values(): - ordered_set.discard(item) - - def get_leftmost(self, key): - return self._items[key].first() - - def get_all(self, key): - return list(self._items[key]) - - def keys(self): - return (k for k, v in self._items.items() if v) - - -class OrderedSet(collections.MutableSet): - def __init__(self): - self._map = collections.OrderedDict() - - def __contains__(self, item): - return item in self._map - - def __iter__(self): - return iter(self._map.keys()) - - def __getitem__(self, ix): - return - - def __len__(self): - return len(self._map) - - def add(self, item): - self._map[item] = None - - def discard(self, item): - try: - del self._map[item] - except KeyError: - pass - - def first(self): - return next(iter(self)) From bf2a93b87e2e1d32bdcf65a8711b88ce471ab081 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 5 Oct 2015 19:58:51 +0300 Subject: [PATCH 081/118] Changed how awaiting multiple methods are handled --- src/asynqp/routing.py | 31 +++++++++++++++++-------------- test/queue_tests.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index c03aa92..fcc61ac 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -75,28 +75,31 @@ def await(self, *expected_methods): return fut for method in expected_methods: - self._futures[method].append((fut, expected_methods)) + self._futures[method].append(fut) return fut def notify(self, method, result=None): - try: - fut, bound_methods = self._futures[method].popleft() - except IndexError: - # XXX: we can't just ignore this. - log.error("Got an unexpected method notification %s", method) - else: - # Cleanup futures, that were awaited together, like - # (spec.BasicGetOK, spec.BasicGetEmpty) - for other_method in bound_methods: - if other_method != method: - self._futures[other_method].remove((fut, bound_methods)) - fut.set_result(result) + while True: + try: + fut = self._futures[method].popleft() + except IndexError: + # XXX: we can't just ignore this. + log.error("Got an unexpected method notification %s", method) + return + # We can have done futures if they were awaited together, like + # (spec.BasicGetOK, spec.BasicGetEmpty). + if not fut.done(): + break + + fut.set_result(result) def killall(self, exc): self.connection_exc = exc # Set an exception for all others for method, futs in self._futures.items(): - for fut, _ in futs: + for fut in futs: + if fut.done(): + continue fut.set_exception(exc) self._futures.clear() diff --git a/test/queue_tests.py b/test/queue_tests.py index e367aba..7cfe755 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -178,6 +178,19 @@ def it_should_put_the_routing_key_on_the_msg(self): assert self.task.result().routing_key == 'routing.key' +class WhenConnectionClosedOnGet(QueueContext): + def given_I_asked_for_a_message(self): + self.task = asyncio.async(self.queue.get(no_ack=False)) + self.tick() + + def when_connection_is_closed(self): + self.server.protocol.connection_lost(None) + self.tick() + + def it_should_raise_exception(self): + assert self.task.exception() is not None + + class WhenISubscribeToAQueue(QueueContext): def when_I_start_a_consumer(self): self.async_partial(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) From 78ad162ca5d5dfd847264555d44b7bec5490bc53 Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 17 Oct 2015 17:06:47 +0300 Subject: [PATCH 082/118] Changed how protocol handles connection_lost. Don't dispatch PoisonPill --- src/asynqp/exceptions.py | 5 ++- src/asynqp/protocol.py | 38 +++++++++++---------- test/channel_tests.py | 11 ++++++ test/connection_tests.py | 11 ++++++ test/heartbeat_tests.py | 15 ++++++--- test/integration_tests.py | 71 ++++++--------------------------------- test/queue_tests.py | 28 +++++++++++++-- 7 files changed, 94 insertions(+), 85 deletions(-) diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 8968466..2f8632f 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -20,7 +20,10 @@ class ConnectionLostError(AlreadyClosed, ConnectionError): ''' Connection was closed unexpectedly ''' - pass + + def __init__(self, message, exc=None): + super().__init__(message) + self.original_exc = exc class UndeliverableMessage(ValueError): diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 3644411..17521c6 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -3,6 +3,7 @@ from . import spec from . import frames from .exceptions import AMQPError, ConnectionLostError +from .log import log class AMQP(asyncio.Protocol): @@ -46,14 +47,23 @@ def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) def connection_lost(self, exc): - if exc is None: - poison_exc = ConnectionLostError( - 'The connection was unexpectedly lost') - else: - poison_exc = exc - self.dispatcher.dispatch_all(frames.PoisonPillFrame(poison_exc)) + # If we exc is None - we closed the transport ourselves. No need to + # dispatch PoisonPillFrame, as we should have closed everything already if exc is not None: - raise ConnectionLostError('The connection was unexpectedly lost') from exc + poison_exc = ConnectionLostError( + 'The connection was unexpectedly lost', exc) + self.dispatcher.dispatch_all(frames.PoisonPillFrame(poison_exc)) + # XXX: Really do we even need to raise this??? It's super bad API + raise poison_exc from exc + + def heartbeat_timeout(self): + """ Called by heartbeat_monitor on timeout """ + log.error("Heartbeat time out") + poison_exc = ConnectionLostError('Heartbeat timed out') + poison_frame = frames.PoisonPillFrame(poison_exc) + self.dispatcher.dispatch_all(poison_frame) + # Spec says to just close socket without ConnectionClose handshake. + self.transport.close() class FrameReader(object): @@ -131,21 +141,15 @@ def monitor_heartbeat(self, interval): self._last_received = self.loop.time() no_beat_for = 0 while True: - # We use interval roundtrip so 2x + # As spec states: + # If a peer detects no incoming traffic (i.e. received octets) for + # two heartbeat intervals or longer, it should close the connection yield from asyncio.sleep( interval * 2 - no_beat_for, loop=self.loop) no_beat_for = self.loop.time() - self._last_received if no_beat_for > interval * 2: - self.protocol.send_method( - 0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) - # It's raised for backward compatibility - try: - self.protocol.connection_lost( - ConnectionLostError('Heartbeat timed out')) - except ConnectionLostError: - pass - break + self.protocol.heartbeat_timeout() def heartbeat_received(self): self._last_received = self.loop.time() diff --git a/test/channel_tests.py b/test/channel_tests.py index 45ce5ad..dbfbd91 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -231,3 +231,14 @@ def when_I_set_the_handler(self): def it_should_throw_a_TypeError(self): assert isinstance(self.exception, TypeError) + + +class WhenAConnectionIsLostCloseChannel(OpenChannelContext): + def when_connection_is_closed(self): + try: + self.connection.protocol.connection_lost(Exception()) + except Exception: + pass + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) diff --git a/test/connection_tests.py b/test/connection_tests.py index b16077d..3909a97 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -105,3 +105,14 @@ def when_another_frame_arrives(self): def it_MUST_be_discarded(self): self.server.should_not_have_received_any() + + +class WhenAConnectionIsLostCloseConnection(OpenConnectionContext): + def when_connection_is_closed(self): + try: + self.connection.protocol.connection_lost(Exception()) + except Exception: + pass + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index 9fe7574..7a6965e 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -1,6 +1,7 @@ import asyncio -from asynqp import spec +from unittest import mock from asynqp.frames import HeartbeatFrame +from asynqp.exceptions import ConnectionLostError from .base_contexts import MockServerContext @@ -43,10 +44,14 @@ def given_i_started_heartbeating(self): self.protocol.start_heartbeat(0.01) def when_the_server_dies(self): - self.loop.run_until_complete(asyncio.sleep(0.021)) - - def it_should_close_the_connection(self): - self.server.should_have_received_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + with mock.patch("asynqp.routing.Dispatcher.dispatch_all") as mocked: + self.loop.run_until_complete(asyncio.sleep(0.021)) + self.mocked = mocked + + def it_should_dispatch_a_poison_pill(self): + assert self.mocked.called + assert isinstance( + self.mocked.call_args[0][0].exception, ConnectionLostError) def cleanup_tasks(self): self.protocol.heartbeat_monitor.stop() diff --git a/test/integration_tests.py b/test/integration_tests.py index f7175e0..b950837 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -3,6 +3,7 @@ import socket import contexts from asyncio import test_utils +from unittest import mock class ConnectionContext: @@ -225,66 +226,6 @@ def it_should_not_throw(self): assert self.exception is None -class WhenAConnectionIsClosedCloseConnection: - def given_a_connection(self): - self.loop = asyncio.get_event_loop() - self.connection = self.loop.run_until_complete(asynqp.connect()) - - def when_connection_is_closed(self): - self.connection.transport.close() - - def it_should_not_hang(self): - self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) - - -class WhenAConnectionIsClosedCloseChannel: - def given_a_channel(self): - self.loop = asyncio.get_event_loop() - self.connection = self.loop.run_until_complete(asynqp.connect()) - self.channel = self.loop.run_until_complete(self.connection.open_channel()) - - def when_connection_is_closed(self): - self.connection.transport.close() - - def it_should_not_hang(self): - self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) - - -class WhenAConnectionIsClosedCancelConsuming: - def given_a_consumer(self): - asynqp.routing._TEST = True - self.loop = asyncio.get_event_loop() - self.connection = self.loop.run_until_complete(asynqp.connect()) - self.channel = self.loop.run_until_complete(self.connection.open_channel()) - self.exchange = self.loop.run_until_complete( - self.channel.declare_exchange(name='name', - type='direct', - durable=False, - auto_delete=True)) - - self.queue = self.loop.run_until_complete( - self.channel.declare_queue(name='', - durable=False, - exclusive=True, - auto_delete=True)) - - self.loop.run_until_complete(self.queue.bind(self.exchange, - 'name')) - - self.consumer = self.loop.run_until_complete( - self.queue.consume(lambda x: x, exclusive=True) - ) - - def when_connection_is_closed(self): - self.connection.transport.close() - - def it_should_not_hang(self): - self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) - - def cleanup(self): - asynqp.routing._TEST = False - - class WhenPublishingWithUnsetLoop: def given_I_have_a_queue(self): @@ -363,3 +304,13 @@ def tear_down(): yield from self.connection.close() self.loop.run_until_complete(tear_down()) asyncio.set_event_loop(self.loop) + + +class WhenAConnectionClosedByHandshakeProtocolShouldNotDispatchPoisonPill(ConnectionContext): + def when_we_close_connection(self): + with mock.patch("asynqp.routing.Dispatcher.dispatch_all") as mocked: + self.loop.run_until_complete(self.connection.close()) + self.mocked = mocked + + def it_should_not_dispatch_poison(self): + assert not self.mocked.called diff --git a/test/queue_tests.py b/test/queue_tests.py index 7cfe755..c347dea 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -184,7 +184,11 @@ def given_I_asked_for_a_message(self): self.tick() def when_connection_is_closed(self): - self.server.protocol.connection_lost(None) + # XXX: remove if we change behaviour to not raise + try: + self.server.protocol.connection_lost(Exception()) + except Exception: + pass self.tick() def it_should_raise_exception(self): @@ -322,7 +326,7 @@ def when_the_connection_dies(self): self.tick() def it_should_call_on_error(self): - assert self.consumer.exc is self.exception + assert self.consumer.exc.original_exc is self.exception class ConsumerWithOnError: def __init__(self): @@ -388,3 +392,23 @@ def when_I_try_to_use_the_queue(self): def it_should_throw_Deleted(self): assert isinstance(self.task.exception(), asynqp.Deleted) + + +class WhenAConnectionIsClosedCancelConsuming(QueueContext, ExchangeContext): + def given_a_consumer(self): + task = asyncio.async(self.queue.consume( + lambda x: None, no_local=False, no_ack=False, + exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.consumer = task.result() + + def when_connection_is_closed(self): + try: + self.connection.protocol.connection_lost(Exception()) + except Exception: + pass + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) From a3cb1687be2a5dda38297f0075c4f01a48935748 Mon Sep 17 00:00:00 2001 From: Taras Date: Sat, 31 Oct 2015 23:57:02 +0200 Subject: [PATCH 083/118] Fix #57 Components not killed when rabbit just closes transport Minor fix for heartbeat to not cancel more than 1 time. --- src/asynqp/connection.py | 4 ++-- src/asynqp/protocol.py | 15 ++++++++++++--- test/base_contexts.py | 4 ++++ test/connection_tests.py | 25 ++++++++++++++++++++++++- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 7eea001..8b25703 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -143,11 +143,11 @@ def handle_ConnectionOpenOK(self, frame): def handle_ConnectionClose(self, frame): self.closing.set_result(True) self.sender.send_CloseOK() - self.protocol.transport.close() + self.protocol.close() self.connection.closed.set_result(True) def handle_ConnectionCloseOK(self, frame): - self.protocol.transport.close() + self.protocol.close() self.synchroniser.notify(spec.ConnectionCloseOK) diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 17521c6..aa15250 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -12,6 +12,7 @@ def __init__(self, dispatcher, loop): self.partial_frame = b'' self.frame_reader = FrameReader() self.heartbeat_monitor = HeartbeatMonitor(self, loop) + self._closed = False def connection_made(self, transport): self.transport = transport @@ -23,7 +24,7 @@ def data_received(self, data): try: result = self.frame_reader.read_frame(data) except AMQPError: - self.transport.close() + self.close() raise if result is None: # incomplete frame, wait for the rest @@ -47,9 +48,9 @@ def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) def connection_lost(self, exc): - # If we exc is None - we closed the transport ourselves. No need to + # If self._closed=True - we closed the transport ourselves. No need to # dispatch PoisonPillFrame, as we should have closed everything already - if exc is not None: + if not self._closed: poison_exc = ConnectionLostError( 'The connection was unexpectedly lost', exc) self.dispatcher.dispatch_all(frames.PoisonPillFrame(poison_exc)) @@ -58,11 +59,17 @@ def connection_lost(self, exc): def heartbeat_timeout(self): """ Called by heartbeat_monitor on timeout """ + assert not self._closed, "Did we not stop heartbeat_monitor on close?" log.error("Heartbeat time out") poison_exc = ConnectionLostError('Heartbeat timed out') poison_frame = frames.PoisonPillFrame(poison_exc) self.dispatcher.dispatch_all(poison_frame) # Spec says to just close socket without ConnectionClose handshake. + self.close() + + def close(self): + assert not self._closed, "Why do we close it 2-ce?" + self._closed = True self.transport.close() @@ -150,6 +157,8 @@ def monitor_heartbeat(self, interval): no_beat_for = self.loop.time() - self._last_received if no_beat_for > interval * 2: self.protocol.heartbeat_timeout() + self.send_hb_task.cancel() + return def heartbeat_received(self): self._last_received = self.loop.time() diff --git a/test/base_contexts.py b/test/base_contexts.py index ad967a3..91f5f67 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -40,6 +40,10 @@ def async_partial(self, coro): self.tick() return t + def wait_for(self, coro): + return self.loop.run_until_complete( + asyncio.wait_for(coro, timeout=0.2, loop=self.loop)) + class MockServerContext(LoopContext): def given_a_mock_server_on_the_other_end_of_the_transport(self): diff --git a/test/connection_tests.py b/test/connection_tests.py index 3909a97..7868977 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,6 +1,6 @@ import asyncio import sys -from asynqp import spec +from asynqp import spec, exceptions from asynqp.connection import open_connection from .base_contexts import MockServerContext, OpenConnectionContext @@ -116,3 +116,26 @@ def when_connection_is_closed(self): def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) + + +class WhenServerClosesTransportWithoutConnectionClose(OpenConnectionContext): + + def given_a_channel(self): + task = self.loop.create_task(self.connection.open_channel()) + self.tick() + self.server.send_method(1, spec.ChannelOpenOK('')) + self.channel = self.wait_for(task) + + def when_server_closes_transport(self): + try: + self.protocol.connection_lost(None) + except exceptions.ConnectionLostError: + pass + + def it_should_raise_error_in_connection_methods(self): + try: + self.wait_for(self.channel.declare_queue("some.queue")) + except exceptions.ConnectionLostError as err: + assert type(err) == exceptions.ConnectionLostError + else: + assert False, "ConnectionLostError not raised" From 78fac49e67e906752613810d5e1f164c97c3aa9b Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 2 Nov 2015 20:35:24 +0200 Subject: [PATCH 084/118] Added passive argument to declare_queue --- src/asynqp/channel.py | 11 +++++++---- src/asynqp/queue.py | 13 ++++++++----- test/queue_tests.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index fea0625..e8d71e3 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -72,7 +72,8 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter return ex @asyncio.coroutine - def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=False, arguments=None): + def declare_queue(self, name='', *, durable=True, exclusive=False, + auto_delete=False, passive=False, arguments=None): """ Declare a queue on the broker. If the queue does not exist, it will be created. @@ -89,7 +90,9 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=F :return: The new :class:`Queue` object. """ - q = yield from self.queue_factory.declare(name, durable, exclusive, auto_delete, arguments if arguments is not None else {}) + q = yield from self.queue_factory.declare( + name, durable, exclusive, auto_delete, passive, + arguments if arguments is not None else {}) return q @asyncio.coroutine @@ -371,8 +374,8 @@ def send_ExchangeDeclare(self, name, type, durable, auto_delete, internal, argum def send_ExchangeDelete(self, name, if_unused): self.send_method(spec.ExchangeDelete(0, name, if_unused, False)) - def send_QueueDeclare(self, name, durable, exclusive, auto_delete, arguments): - self.send_method(spec.QueueDeclare(0, name, False, durable, exclusive, auto_delete, False, arguments)) + def send_QueueDeclare(self, name, durable, exclusive, auto_delete, passive, arguments): + self.send_method(spec.QueueDeclare(0, name, passive, durable, exclusive, auto_delete, False, arguments)) def send_QueueBind(self, queue_name, exchange_name, routing_key, arguments): self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, arguments)) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index fee353b..1bf9a17 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -280,13 +280,16 @@ def __init__(self, sender, synchroniser, reader, consumers, *, loop): self.consumers = consumers @asyncio.coroutine - def declare(self, name, durable, exclusive, auto_delete, arguments): + def declare(self, name, durable, exclusive, auto_delete, passive, + arguments): if not VALID_QUEUE_NAME_RE.match(name): - raise ValueError("Not a valid queue name.\n" - "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " - "and do not begin with 'amq.'") + raise ValueError( + "Not a valid queue name.\n" + "Valid names consist of letters, digits, hyphen, underscore, " + "period, or colon, and do not begin with 'amq.'") - self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete, arguments) + self.sender.send_QueueDeclare( + name, durable, exclusive, auto_delete, passive, arguments) name = yield from self.synchroniser.await(spec.QueueDeclareOK) q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete, arguments, diff --git a/test/queue_tests.py b/test/queue_tests.py index c347dea..8033ef9 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -5,6 +5,7 @@ from asynqp import message from asynqp import frames from asynqp import spec +from asynqp import exceptions from .base_contexts import OpenChannelContext, QueueContext, ExchangeContext, BoundQueueContext, ConsumerContext from .util import testing_exception_handler @@ -412,3 +413,40 @@ def when_connection_is_closed(self): def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) + + +class WhenIDeclareQueueWithPassiveAndOKArrives(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.async(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=True, + passive=True), loop=self.loop) + self.tick() + + def when_QueueDeclareOK_arrives(self): + self.server.send_method( + self.channel.id, spec.QueueDeclareOK('123', 123, 456)) + + def it_should_return_queue_object(self): + result = self.task.result() + assert result + assert result.name == '123' + + +class WhenIDeclareQueueWithPassiveAndErrorArrives(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.async(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=True, + passive=True), loop=self.loop) + self.tick() + + def when_QueueDeclareOK_arrives(self): + self.server.send_method( + self.channel.id, spec.ChannelClose(404, 'Bad queue', 40, 50)) + + def it_should_raise_exception(self): + try: + self.task.result() + except exceptions.NotFound: + pass + else: + assert False, "NotFound exception not raised" From 34ec8c67540ab48161ea84d0d5fc1470c9cbe08b Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 2 Nov 2015 20:51:14 +0200 Subject: [PATCH 085/118] Added passive argument to declare exchange --- src/asynqp/channel.py | 24 ++++++++++++++--------- test/exchange_tests.py | 44 ++++++++++++++++++++++++++++++++++++++++++ test/queue_tests.py | 9 +++++++-- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index e8d71e3..c2e75d3 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -39,7 +39,8 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factor self.reader = reader @asyncio.coroutine - def declare_exchange(self, name, type, *, durable=True, auto_delete=False, internal=False, arguments=None): + def declare_exchange(self, name, type, *, durable=True, auto_delete=False, + passive=False, internal=False, arguments=None): """ Declare an :class:`Exchange` on the broker. If the exchange does not exist, it will be created. @@ -61,13 +62,18 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter return exchange.Exchange(self.reader, self.synchroniser, self.sender, name, 'direct', True, False, False) if not VALID_EXCHANGE_NAME_RE.match(name): - raise ValueError("Invalid exchange name.\n" - "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " - "and do not begin with 'amq.'") - - self.sender.send_ExchangeDeclare(name, type, durable, auto_delete, internal, arguments or {}) + raise ValueError( + "Invalid exchange name.\n" + "Valid names consist of letters, digits, hyphen, underscore, " + "period, or colon, and do not begin with 'amq.'") + + self.sender.send_ExchangeDeclare( + name, type, passive, durable, auto_delete, internal, + arguments or {}) yield from self.synchroniser.await(spec.ExchangeDeclareOK) - ex = exchange.Exchange(self.reader, self.synchroniser, self.sender, name, type, durable, auto_delete, internal) + ex = exchange.Exchange( + self.reader, self.synchroniser, self.sender, name, type, durable, + auto_delete, internal) self.reader.ready() return ex @@ -368,8 +374,8 @@ def __init__(self, channel_id, protocol, connection_info): def send_ChannelOpen(self): self.send_method(spec.ChannelOpen('')) - def send_ExchangeDeclare(self, name, type, durable, auto_delete, internal, arguments): - self.send_method(spec.ExchangeDeclare(0, name, type, False, durable, auto_delete, internal, False, arguments)) + def send_ExchangeDeclare(self, name, type, passive, durable, auto_delete, internal, arguments): + self.send_method(spec.ExchangeDeclare(0, name, type, passive, durable, auto_delete, internal, False, arguments)) def send_ExchangeDelete(self, name, if_unused): self.send_method(spec.ExchangeDelete(0, name, if_unused, False)) diff --git a/test/exchange_tests.py b/test/exchange_tests.py index 72ca7db..451eae8 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -5,6 +5,7 @@ from asynqp import spec from asynqp import frames from asynqp import message +from asynqp import exceptions from .base_contexts import OpenChannelContext, ExchangeContext @@ -174,3 +175,46 @@ def when_confirmation_arrives(self): def it_should_not_throw(self): pass + + +class WhenExchangeDeclareWithPassiveAndOKArrives(OpenChannelContext): + def given_I_declared_an_exchange(self): + self.task = asyncio.async( + self.channel.declare_exchange( + 'name_1', 'fanout', passive=True, + durable=True, auto_delete=False, internal=False)) + self.tick() + + def when_the_exchange_declare_ok_arrives(self): + self.server.send_method(self.channel.id, spec.ExchangeDeclareOK()) + + def it_should_return_an_exchange_object(self): + result = self.task.result() + assert result.name == 'name_1' + assert result.type == 'fanout' + + def it_should_have_sent_passive_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.ExchangeDeclare( + 0, 'name_1', 'fanout', True, True, False, False, False, {})) + + +class WhenExchangeDeclareWithPassiveAndErrorArrives(OpenChannelContext): + def given_I_declared_an_exchange(self): + self.task = asyncio.async( + self.channel.declare_exchange( + 'name_1', 'fanout', passive=True, + durable=True, auto_delete=False, internal=False)) + self.tick() + + def when_error_arrives(self): + self.server.send_method( + self.channel.id, spec.ChannelClose(404, 'Bad exchange', 40, 50)) + + def it_should_return_an_exchange_object(self): + try: + self.task.result() + except exceptions.NotFound: + pass + else: + assert False, "NotFound exception not raised" diff --git a/test/queue_tests.py b/test/queue_tests.py index 8033ef9..63beba7 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -418,7 +418,7 @@ def it_should_not_hang(self): class WhenIDeclareQueueWithPassiveAndOKArrives(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): self.task = asyncio.async(self.channel.declare_queue( - '123', durable=True, exclusive=True, auto_delete=True, + '123', durable=True, exclusive=True, auto_delete=False, passive=True), loop=self.loop) self.tick() @@ -431,6 +431,11 @@ def it_should_return_queue_object(self): assert result assert result.name == '123' + def it_should_have_sent_passive_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.QueueDeclare( + 0, '123', True, True, True, False, False, {})) + class WhenIDeclareQueueWithPassiveAndErrorArrives(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): @@ -439,7 +444,7 @@ def given_I_declared_a_queue_with_passive(self): passive=True), loop=self.loop) self.tick() - def when_QueueDeclareOK_arrives(self): + def when_error_arrives(self): self.server.send_method( self.channel.id, spec.ChannelClose(404, 'Bad queue', 40, 50)) From 9054fe0a5992071556ff41cfa6d14e99ef93cdb9 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 2 Nov 2015 21:20:41 +0200 Subject: [PATCH 086/118] Added docs for passive arguments --- src/asynqp/channel.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index c2e75d3..84af0bc 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -49,12 +49,18 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, :param str name: the name of the exchange. :param str type: the type of the exchange (usually one of ``'fanout'``, ``'direct'``, ``'topic'``, or ``'headers'``) - :keyword bool durable: If true, the exchange will be re-created when the server restarts. + :keyword bool durable: If true, the exchange will be re-created when + the server restarts. :keyword bool auto_delete: If true, the exchange will be deleted when the last queue is un-bound from it. - :keyword bool internal: If true, the exchange cannot be published to directly; - it can only be bound to other exchanges. - :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. + :keyword bool passive: If `true` and exchange with such a name does + not exist it will raise a :class:`exceptions.NotFound`. If `false` + server will create it. Arguments ``durable``, ``auto_delete`` and + ``internal`` are ignored if `passive=True`. + :keyword bool internal: If true, the exchange cannot be published to + directly; it can only be bound to other exchanges. + :keyword dict arguments: Table of optional parameters for extensions to + the AMQP protocol. See :ref:`extensions`. :return: the new :class:`Exchange` object. """ @@ -92,6 +98,10 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, and will be deleted when the connection is closed. :keyword bool auto_delete: If true, the queue will be deleted when the last consumer is cancelled. If there were never any conusmers, the queue won't be deleted. + :keyword bool passive: If true and queue with such a name does not + exist it will raise a :class:`exceptions.NotFound` instead of + creating it. Arguments ``durable``, ``auto_delete`` and + ``exclusive`` are ignored if ``passive=True``. :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`Queue` object. From 95551f71efa660650759c12f3d6027fe8cdd4801 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 3 Nov 2015 11:34:31 +0000 Subject: [PATCH 087/118] fix #58 by raising AlreadyClosed from the connection --- src/asynqp/connection.py | 2 ++ test/base_contexts.py | 3 +-- test/connection_tests.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 8b25703..ba1e59c 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -54,6 +54,8 @@ def open_channel(self): :return: The new :class:`Channel` object. """ + if self.closed.done(): + raise AlreadyClosed() channel = yield from self.channel_factory.open() return channel diff --git a/test/base_contexts.py b/test/base_contexts.py index 91f5f67..ef84af5 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -41,8 +41,7 @@ def async_partial(self, coro): return t def wait_for(self, coro): - return self.loop.run_until_complete( - asyncio.wait_for(coro, timeout=0.2, loop=self.loop)) + return self.loop.run_until_complete(asyncio.wait_for(coro, timeout=0.2, loop=self.loop)) class MockServerContext(LoopContext): diff --git a/test/connection_tests.py b/test/connection_tests.py index 7868977..8c7b635 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,5 +1,6 @@ import asyncio import sys +import contexts from asynqp import spec, exceptions from asynqp.connection import open_connection from .base_contexts import MockServerContext, OpenConnectionContext @@ -139,3 +140,16 @@ def it_should_raise_error_in_connection_methods(self): assert type(err) == exceptions.ConnectionLostError else: assert False, "ConnectionLostError not raised" + + +class WhenOpeningAChannelOnAClosedConnection(OpenConnectionContext): + def when_client_closes_connection(self): + task = asyncio.async(self.connection.close()) + self.tick() + self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() + task.result() + + def it_should_raise_error_in_connection_methods(self): + exc = contexts.catch(self.wait_for, self.connection.open_channel()) + assert isinstance(exc, exceptions.AlreadyClosed) From 5fbde91f640b81485b3b594d99f979e03dcd1d51 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Tue, 3 Nov 2015 11:39:22 +0000 Subject: [PATCH 088/118] mute log in tests --- test/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/__init__.py b/test/__init__.py index 5dbd8ac..96ddb0e 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,6 +1,10 @@ import asyncio +import logging from .util import testing_exception_handler loop = asyncio.get_event_loop() loop.set_exception_handler(testing_exception_handler) + + +logging.getLogger('asynqp').setLevel(100) # mute the logger From abd7245c0950047bd93f06e4528c9aa1b585983e Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 3 Nov 2015 20:16:41 +0200 Subject: [PATCH 089/118] Little cleanup for passive argument tests. --- test/exchange_tests.py | 9 ++------- test/queue_tests.py | 7 +------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/test/exchange_tests.py b/test/exchange_tests.py index 451eae8..ff7161b 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -211,10 +211,5 @@ def when_error_arrives(self): self.server.send_method( self.channel.id, spec.ChannelClose(404, 'Bad exchange', 40, 50)) - def it_should_return_an_exchange_object(self): - try: - self.task.result() - except exceptions.NotFound: - pass - else: - assert False, "NotFound exception not raised" + def it_should_raise_not_found_error(self): + assert isinstance(self.task.exception(), exceptions.NotFound) diff --git a/test/queue_tests.py b/test/queue_tests.py index 63beba7..f7ccdce 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -449,9 +449,4 @@ def when_error_arrives(self): self.channel.id, spec.ChannelClose(404, 'Bad queue', 40, 50)) def it_should_raise_exception(self): - try: - self.task.result() - except exceptions.NotFound: - pass - else: - assert False, "NotFound exception not raised" + assert isinstance(self.task.exception(), exceptions.NotFound) From 4bf0470c89439f8c3fc05776dfb4f8eac68dae04 Mon Sep 17 00:00:00 2001 From: Taras Date: Wed, 30 Sep 2015 19:41:37 +0300 Subject: [PATCH 090/118] Refactor close procedure of Connection, Channel and Consumer to raise good grained exceptions. Changed Exceptions hierarchy to: AMQPError AMQPConnectionError ConnectionLostError ClientConnectionClosed ServerConnectionClosed AMQPChannelError ClientChannelClosed ContentTooLarge NotFound NoConsumers NotAllowed ... --- .gitignore | 1 + src/asynqp/__init__.py | 8 ++- src/asynqp/_exceptions.py | 4 ++ src/asynqp/channel.py | 129 +++++++++++++++++++++++++++----------- src/asynqp/connection.py | 80 ++++++++++++++++------- src/asynqp/exceptions.py | 30 ++++++--- src/asynqp/protocol.py | 3 + src/asynqp/queue.py | 6 +- src/asynqp/routing.py | 33 ++++++---- test/base_contexts.py | 8 +-- test/channel_tests.py | 53 ++++++++++++++++ test/connection_tests.py | 30 ++++++--- 12 files changed, 288 insertions(+), 97 deletions(-) diff --git a/.gitignore b/.gitignore index 345208f..c0a52f3 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ doc/_build/ *.sublime-workspace .ipynb_checkpoints +.DS_Store diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index dddbf08..9b101b5 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -69,7 +69,13 @@ def connect(host='localhost', if (sk.family in (socket.AF_INET, socket.AF_INET6)) and (sk.proto in (0, socket.IPPROTO_TCP)): sk.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - connection = yield from open_connection(loop, transport, protocol, dispatcher, {'username': username, 'password': password, 'virtual_host': virtual_host}) + connection_info = { + 'username': username, + 'password': password, + 'virtual_host': virtual_host + } + connection = yield from open_connection( + loop, transport, protocol, dispatcher, connection_info) return connection diff --git a/src/asynqp/_exceptions.py b/src/asynqp/_exceptions.py index dcf784c..fee3757 100644 --- a/src/asynqp/_exceptions.py +++ b/src/asynqp/_exceptions.py @@ -1,2 +1,6 @@ class AMQPError(IOError): pass + + +class AMQPChannelError(AMQPError): + pass diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index fea0625..92f9c4e 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -8,7 +8,10 @@ from . import exchange from . import message from . import routing -from .exceptions import UndeliverableMessage, AlreadyClosed +from .exceptions import ( + UndeliverableMessage, ClientChannelClosed, + ServerConnectionClosed, ClientConnectionClosed, AMQPError) +from .log import log VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -29,7 +32,8 @@ class Channel(object): the numerical ID of the channel """ - def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader, *, loop): + def __init__(self, id, synchroniser, sender, basic_return_consumer, + queue_factory, reader, *, loop): self._loop = loop self.id = id self.synchroniser = synchroniser @@ -37,6 +41,8 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factor self.basic_return_consumer = basic_return_consumer self.queue_factory = queue_factory self.reader = reader + # Indicates, that channel is closed or started closing + self._closing = False @asyncio.coroutine def declare_exchange(self, name, type, *, durable=True, auto_delete=False, internal=False, arguments=None): @@ -92,21 +98,6 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=F q = yield from self.queue_factory.declare(name, durable, exclusive, auto_delete, arguments if arguments is not None else {}) return q - @asyncio.coroutine - def close(self): - """ - Close the channel by handshaking with the server. - - This method is a :ref:`coroutine `. - """ - self._closing.set_result(True) - self.sender.send_Close(0, 'Channel closed by application', 0, 0) - try: - yield from self.synchroniser.await(spec.ChannelCloseOK) - except AlreadyClosed: - pass - # don't call self.reader.ready - stop reading frames from the q - @asyncio.coroutine def set_qos(self, prefetch_size=0, prefetch_count=0, apply_globally=False): """ @@ -151,6 +142,30 @@ def set_return_handler(self, handler): """ self.basic_return_consumer.set_callback(handler) + @asyncio.coroutine + def close(self): + """ + Close the channel by handshaking with the server. + + This method is a :ref:`coroutine `. + """ + # If we aren't already closed ask for server to close + if not self._closing: + self._closing = True + # Let the ChannelActor do the actual close operations. + # It will do the work on CloseOK + try: + self.sender.send_Close( + 0, 'Channel closed by application', 0, 0) + yield from self.synchroniser.await(spec.ChannelCloseOK) + except AMQPError: + # For example if both sides want to close or the connection + # is closed. + pass + else: + # Why are we closing a closed channel? + log.warn("Called `close` on already closed channel...") + class ChannelFactory(object): def __init__(self, loop, protocol, dispatcher, connection_info): @@ -171,16 +186,19 @@ def open(self): consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - actor = ChannelActor(consumers, synchroniser, sender, loop=self.loop) + actor = ChannelActor(synchroniser, sender, loop=self.loop) reader = routing.QueuedReader(actor, loop=self.loop) - actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) queue_factory = queue.QueueFactory( sender, synchroniser, reader, consumers, loop=self.loop) channel = Channel( channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader, loop=self.loop) - channel._closing = actor.closing + + # Set actor dependencies + actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) + actor.consumers = consumers + actor.channel = channel self.dispatcher.add_handler(channel_id, reader.feed) try: @@ -203,13 +221,12 @@ def open(self): class ChannelActor(routing.Actor): - def __init__(self, consumers, *args, **kwargs): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.consumers = consumers - - def handle_PoisonPillFrame(self, frame): - super().handle_PoisonPillFrame(frame) - self.consumers.error(frame.exception) + # Will set those in Channel factory + self.channel = None + self.consumers = None + self.message_receiver = None def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) @@ -245,14 +262,6 @@ def handle_BasicConsumeOK(self, frame): def handle_BasicCancelOK(self, frame): self.synchroniser.notify(spec.BasicCancelOK) - def handle_ChannelClose(self, frame): - self.sender.send_CloseOK() - exc = exceptions._get_exception_type(frame.payload.reply_code) - self.synchroniser.killall(exc) - - def handle_ChannelCloseOK(self, frame): - self.synchroniser.notify(spec.ChannelCloseOK) - def handle_BasicQosOK(self, frame): self.synchroniser.notify(spec.BasicQosOK) @@ -279,6 +288,57 @@ def handle_BasicReturn(self, frame): assert self.message_receiver is not None, "message_receiver not set" self.message_receiver.receive_return(frame) + # Close handlers + + def handle_PoisonPillFrame(self, frame): + """ Is sent in case protocol lost connection to server.""" + # Make sure all `close` calls don't deadlock + self.channel._closing = True + + exc = frame.exception + self._close_all(exc) + + def handle_ChannelClose(self, frame): + """ AMQP server closed the channel with an error """ + # Tell server we understood and close + self.sender.send_CloseOK() + # Make sure all `close` calls don't deadlock + self.channel._closing = True + exc = exceptions._get_exception_type(frame.payload.reply_code) + self._close_all(exc) + + def handle_ChannelCloseOK(self, frame): + """ AMQP server closed channel as per our request """ + # Let the close method continue working + self.synchroniser.notify(spec.ChannelCloseOK) + exc = ClientChannelClosed() + self._close_all(exc) + + def handle_ConnectionClose(self, frame): + """ AMQP server closed the connection with an error """ + # Make sure all `close` calls don't deadlock + self.channel._closing = True + exc = ServerConnectionClosed() + self._close_all(exc) + + def handle_ConnectionCloseOK(self, frame): + """ AMQP server closed connection as per our request """ + # Make sure all `close` calls don't deadlock + self.channel._closing = True + exc = ClientConnectionClosed() + self._close_all(exc) + + def _close_all(self, exc): + # If there were anyone who expected an `*-OK` kill them, as no data + # will follow after close. Any new calls should also raise an error. + self.synchroniser.killall(exc) + self.sender.killall(exc) + # Cancel all consumers with same error + self.consumers.error(exc) + + # TODO: Add `handle_BasicCancel` for + # https://www.rabbitmq.com/consumer-cancel.html support + class MessageReceiver(object): def __init__(self, synchroniser, sender, consumers, reader): @@ -312,7 +372,6 @@ def receive_deliver(self, frame): payload.routing_key, payload.consumer_tag ) - # Delivers message to consumers when done self.is_getok_message = False self.reader.ready() diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index ba1e59c..639465b 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -1,12 +1,11 @@ import asyncio -import logging import sys -from . import channel from . import spec from . import routing -from .exceptions import AlreadyClosed - -log = logging.getLogger(__name__) +from .channel import ChannelFactory +from .exceptions import ( + AMQPConnectionError, ClientConnectionClosed, ServerConnectionClosed) +from .log import log class Connection(object): @@ -36,14 +35,18 @@ class Connection(object): The :class:`~asyncio.Protocol` which is paired with the transport """ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): + self._loop = loop self.synchroniser = synchroniser self.sender = sender - self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) + self.channel_factory = ChannelFactory(loop, protocol, dispatcher, connection_info) self.connection_info = connection_info self.transport = transport self.protocol = protocol - self.closed = asyncio.Future(loop=loop) + # Indicates, that close was initiated by client + self._closing = False + # List of created channels + self._close_waiters = [] @asyncio.coroutine def open_channel(self): @@ -54,8 +57,8 @@ def open_channel(self): :return: The new :class:`Channel` object. """ - if self.closed.done(): - raise AlreadyClosed() + if self._closing: + raise ClientConnectionClosed() channel = yield from self.channel_factory.open() return channel @@ -66,21 +69,23 @@ def close(self): This method is a :ref:`coroutine `. """ - if not self.closed.done(): - self._closing.set_result(True) - self.sender.send_Close(0, 'Connection closed by application', 0, 0) + if not self._closing: + self._closing = True + # Let the ConnectionActor do the actual close operations. + # It will do the work on CloseOK try: + self.sender.send_Close( + 0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) - except AlreadyClosed: + except AMQPConnectionError: + # For example if both sides want to close or the connection + # is closed. pass - # Close heartbeat - # TODO: We really need a better solution for finalization of parts - # in the library. - self.protocol.heartbeat_monitor.stop() - yield from self.protocol.heartbeat_monitor.wait_closed() - self.closed.set_result(True) else: log.warn("Called `close` on already closed connection...") + # Wait for all components to close + if self._close_waiters: + yield from asyncio.gather(*self._close_waiters, loop=self._loop) @asyncio.coroutine @@ -90,7 +95,6 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): sender = ConnectionMethodSender(protocol) connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) actor = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) - connection._closing = actor.closing # bit ugly reader = routing.QueuedReader(actor, loop=loop) try: @@ -142,15 +146,43 @@ def handle_ConnectionTune(self, frame): def handle_ConnectionOpenOK(self, frame): self.synchroniser.notify(spec.ConnectionOpenOK) + def handle_PoisonPillFrame(self, frame): + """ Is sent in case protocol lost connection to server.""" + # Make sure all `close` calls don't deadlock + self.connection._closing = True + exc = frame.exception + # Transport is already closed + self._close_all(exc) + def handle_ConnectionClose(self, frame): - self.closing.set_result(True) + """ AMQP server closed the channel with an error """ + # Notify server we are OK to close. self.sender.send_CloseOK() - self.protocol.close() - self.connection.closed.set_result(True) + self.connection._closing = True + exc = ServerConnectionClosed() + self._close_all(exc) + + # Don't close transport right away, as CloseOK might not get to server + # yet. At least give the loop a spin before we do so. + # TODO: After FlowControl is implemented change this to drain and close + self._loop.call_soon(self.protocol.close) def handle_ConnectionCloseOK(self, frame): - self.protocol.close() self.synchroniser.notify(spec.ConnectionCloseOK) + exc = ClientConnectionClosed() + self._close_all(exc) + # We already agread with server on closing, so lets do it right away + self.protocol.close() + + def _close_all(self, exc): + # Close heartbeat + self.protocol.heartbeat_monitor.stop() + self.connection._close_waiters.append( + self.protocol.heartbeat_monitor.wait_closed()) + # If there were anyone who expected an `*-OK` kill them, as no data + # will follow after close + self.synchroniser.killall(exc) + self.sender.killall(exc) class ConnectionMethodSender(routing.Sender): diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 2f8632f..e46a38e 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -1,31 +1,45 @@ -from ._exceptions import AMQPError +from ._exceptions import AMQPError, AMQPChannelError from .spec import EXCEPTIONS, CONSTANTS_INVERSE __all__ = [ "AMQPError", "ConnectionLostError", + "ServerConnectionClosed", + "ClientChannelClosed", + "ClientConnectionClosed", + "AMQPChannelError", + "AMQPConnectionError", "UndeliverableMessage", "Deleted" ] __all__.extend(EXCEPTIONS.keys()) -class AlreadyClosed(Exception): - """ Raised when issuing commands on closed Channel/Connection. - """ +class AMQPConnectionError(AMQPError): + pass -class ConnectionLostError(AlreadyClosed, ConnectionError): - ''' - Connection was closed unexpectedly - ''' +class ConnectionLostError(AMQPConnectionError, ConnectionError): + """ Connection was closed unexpectedly """ def __init__(self, message, exc=None): super().__init__(message) self.original_exc = exc +class ClientConnectionClosed(AMQPConnectionError): + """ Connection was closed by client """ + + +class ClientChannelClosed(AMQPChannelError): + """ Channel was closed by client """ + + +class ServerConnectionClosed(AMQPConnectionError): + """ Connection was closed by server """ + + class UndeliverableMessage(ValueError): pass diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index aa15250..2067a2c 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -8,6 +8,7 @@ class AMQP(asyncio.Protocol): def __init__(self, dispatcher, loop): + self._loop = loop self.dispatcher = dispatcher self.partial_frame = b'' self.frame_reader = FrameReader() @@ -139,6 +140,8 @@ def wait_closed(self): @asyncio.coroutine def send_heartbeat(self, interval): + # XXX: Add `last_sent` frame monitoring to not send heartbeats + # if traffic was going through socket while True: self.protocol.send_frame(frames.HeartbeatFrame()) yield from asyncio.sleep(interval, loop=self.loop) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index fee353b..1941ce2 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -2,7 +2,7 @@ import re from operator import delitem from . import spec -from .exceptions import Deleted, AlreadyClosed +from .exceptions import Deleted, AMQPError VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -257,10 +257,10 @@ def cancel(self): This method is a :ref:`coroutine `. """ - self.sender.send_BasicCancel(self.tag) try: + self.sender.send_BasicCancel(self.tag) yield from self.synchroniser.await(spec.BasicCancelOK) - except AlreadyClosed: + except AMQPError: pass else: # No need to call ready if connection closed. diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index fcc61ac..e4908c7 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -5,9 +5,6 @@ from .log import log -_TEST = False - - class Dispatcher(object): def __init__(self): self.handlers = {} @@ -21,8 +18,15 @@ def remove_handler(self, channel_id): def dispatch(self, frame): if isinstance(frame, frames.HeartbeatFrame): return - handler = self.handlers[frame.channel_id] - handler(frame) + # XXX: Could not find a better way to close channels... + elif isinstance(frame.payload, (spec.ConnectionCloseOK, + spec.ConnectionClose)): + # Notify all connection channels about it + for handler in self.handlers.values(): + handler(frame) + else: + handler = self.handlers[frame.channel_id] + handler(frame) def dispatch_all(self, frame): for handler in self.handlers.values(): @@ -33,22 +37,27 @@ class Sender(object): def __init__(self, channel_id, protocol): self.channel_id = channel_id self.protocol = protocol + self.connection_exc = None def send_method(self, method): + if self.connection_exc is not None: + raise self.connection_exc self.protocol.send_method(self.channel_id, method) + def killall(self, exc): + """ Connection/Channel was closed. All subsequent requests should + raise an error + """ + self._connection_exc = exc + class Actor(object): def __init__(self, synchroniser, sender, *, loop): self._loop = loop self.synchroniser = synchroniser self.sender = sender - self.closing = asyncio.Future(loop=self._loop) def handle(self, frame): - close_methods = (spec.ConnectionClose, spec.ConnectionCloseOK, spec.ChannelClose, spec.ChannelCloseOK) - if self.closing.done() and not isinstance(frame.payload, close_methods): - return try: meth = getattr(self, 'handle_' + type(frame).__name__) except AttributeError: @@ -56,9 +65,6 @@ def handle(self, frame): meth(frame) - def handle_PoisonPillFrame(self, frame): - self.synchroniser.killall(frame.exception) - class Synchroniser(object): @@ -94,6 +100,9 @@ def notify(self, method, result=None): fut.set_result(result) def killall(self, exc): + """ Connection/Channel was closed. All subsequent and ongoing requests + should raise an error + """ self.connection_exc = exc # Set an exception for all others for method, futs in self._futures.items(): diff --git a/test/base_contexts.py b/test/base_contexts.py index ef84af5..682f048 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -14,15 +14,9 @@ def given_an_event_loop(self): self.exceptions = [] self.loop = asyncio.get_event_loop() self.loop.set_debug(True) - self.loop.set_exception_handler(self.exception_handler) - asynqp.routing._TEST = True def cleanup_test_hack(self): self.loop.set_debug(False) - self.loop.set_exception_handler(None) - asynqp.routing._TEST = False - if self.exceptions: - raise self.exceptions[0] def exception_handler(self, loop, context): self.exceptions.append(context['exception']) @@ -84,7 +78,7 @@ def open_channel(self, channel_id=1): task = asyncio.async(self.connection.open_channel(), loop=self.loop) self.tick() self.server.send_method(channel_id, spec.ChannelOpenOK('')) - return task.result() + return self.loop.run_until_complete(task) class QueueContext(OpenChannelContext): diff --git a/test/channel_tests.py b/test/channel_tests.py index dbfbd91..2918ff0 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -239,6 +239,59 @@ def when_connection_is_closed(self): self.connection.protocol.connection_lost(Exception()) except Exception: pass + self.tick() + self.was_closed = self.channel._closing def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenWeCloseConnectionChannelShouldAlsoClose(OpenChannelContext): + def when_connection_is_closed(self): + self.task = asyncio.async(self.connection.close(), loop=self.loop) + self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() + self.was_closed = self.channel._closing + self.loop.run_until_complete(asyncio.wait_for(self.task, 0.2)) + + def it_should_not_hang_channel_close(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenServerClosesConnectionChannelShouldAlsoClose(OpenChannelContext): + def when_connection_is_closed(self): + self.server.send_method( + 0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) + self.tick() + self.was_closed = self.channel._closing + + def it_should_not_hang_channel_close(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenServerAndClientCloseChannelAtATime(OpenChannelContext): + def when_both_sides_close_channel(self): + # Client tries to close connection + self.task = asyncio.async(self.channel.close(), loop=self.loop) + self.tick() + # Before OK arrives server closes connection + self.server.send_method( + self.channel.id, + spec.ChannelClose(404, 'i am tired of you', 40, 50)) + self.tick() + self.task.result() + + def if_should_have_closed_channel(self): + assert self.channel._closing + + def it_should_have_killed_synchroniser_with_404(self): + assert self.channel.synchroniser.connection_exc == exceptions.NotFound diff --git a/test/connection_tests.py b/test/connection_tests.py index 8c7b635..ddd60f6 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -46,9 +46,6 @@ def when_the_close_frame_arrives(self): def it_should_send_close_ok(self): self.server.should_have_received_method(0, spec.ConnectionCloseOK()) - def it_should_set_the_future(self): - assert self.connection.closed.done() - def it_should_not_block_clonnection_close(self): self.loop.run_until_complete( asyncio.wait_for(self.connection.close(), 0.2)) @@ -75,9 +72,6 @@ def when_connection_close_ok_arrives(self): def it_should_close_the_transport(self): assert self.transport.closed - def it_should_set_the_future(self): - assert self.connection.closed.done() - class WhenAConnectionThatIsClosingReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): @@ -148,8 +142,30 @@ def when_client_closes_connection(self): self.tick() self.server.send_method(0, spec.ConnectionCloseOK()) self.tick() + self.tick() task.result() def it_should_raise_error_in_connection_methods(self): exc = contexts.catch(self.wait_for, self.connection.open_channel()) - assert isinstance(exc, exceptions.AlreadyClosed) + assert isinstance(exc, exceptions.ClientConnectionClosed) + + +class WhenServerAndClientCloseConnectionAtATime(OpenConnectionContext): + def when_both_sides_close_channel(self): + # Client tries to close connection + self.task = asyncio.async(self.connection.close(), loop=self.loop) + self.tick() + # Before OK arrives server closes connection + self.server.send_method( + 0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) + self.tick() + self.tick() + self.task.result() + + def if_should_have_closed_connection(self): + assert self.connection._closing + + def it_should_have_killed_synchroniser_with_server_error(self): + assert isinstance( + self.connection.synchroniser.connection_exc, + exceptions.ServerConnectionClosed) From 019868709b35481595349f0a6db826ba8cfd622b Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 3 Nov 2015 23:35:07 +0200 Subject: [PATCH 091/118] Cleaned up and fixed tests --- src/asynqp/channel.py | 70 ++++++++++++++++++---------------- src/asynqp/connection.py | 79 ++++++++++++++++++++++++++------------- src/asynqp/exceptions.py | 5 +++ src/asynqp/protocol.py | 1 - src/asynqp/queue.py | 4 +- src/asynqp/routing.py | 21 +---------- test/channel_tests.py | 8 ++-- test/integration_tests.py | 10 ----- 8 files changed, 102 insertions(+), 96 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 92f9c4e..813601e 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -41,6 +41,7 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, self.basic_return_consumer = basic_return_consumer self.queue_factory = queue_factory self.reader = reader + self._closed = False # Indicates, that channel is closed or started closing self._closing = False @@ -142,6 +143,9 @@ def set_return_handler(self, handler): """ self.basic_return_consumer.set_callback(handler) + def is_closed(self): + return self._closing or self._closed + @asyncio.coroutine def close(self): """ @@ -150,21 +154,21 @@ def close(self): This method is a :ref:`coroutine `. """ # If we aren't already closed ask for server to close - if not self._closing: + if not self.is_closed(): self._closing = True # Let the ChannelActor do the actual close operations. # It will do the work on CloseOK + self.sender.send_Close( + 0, 'Channel closed by application', 0, 0) try: - self.sender.send_Close( - 0, 'Channel closed by application', 0, 0) yield from self.synchroniser.await(spec.ChannelCloseOK) except AMQPError: # For example if both sides want to close or the connection # is closed. pass else: - # Why are we closing a closed channel? - log.warn("Called `close` on already closed channel...") + if self._closing: + log.warn("Called `close` on already closing channel...") class ChannelFactory(object): @@ -228,6 +232,20 @@ def __init__(self, *args, **kwargs): self.consumers = None self.message_receiver = None + def handle(self, frame): + # From docs on `close`: + # After sending this method, any received methods except Close and + # Close-OK MUST be discarded. + # So we will only process ChannelClose, ChannelCloseOK, PoisonPillFrame + # if channel is closed + if self.channel.is_closed(): + close_methods = (spec.ChannelClose, spec.ChannelCloseOK) + if isinstance(frame.payload, close_methods) or isinstance(frame, frames.PoisonPillFrame): + return super().handle(frame) + else: + return + return super().handle(frame) + def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) @@ -291,54 +309,39 @@ def handle_BasicReturn(self, frame): # Close handlers def handle_PoisonPillFrame(self, frame): - """ Is sent in case protocol lost connection to server.""" - # Make sure all `close` calls don't deadlock - self.channel._closing = True - - exc = frame.exception - self._close_all(exc) + """ Is sent in case connection was closed or disconnected.""" + self._close_all(frame.exception) def handle_ChannelClose(self, frame): """ AMQP server closed the channel with an error """ - # Tell server we understood and close + # By docs: + # The response to receiving a Close after sending Close must be to + # send Close-Ok. + # + # No need for additional checks + self.sender.send_CloseOK() - # Make sure all `close` calls don't deadlock - self.channel._closing = True exc = exceptions._get_exception_type(frame.payload.reply_code) self._close_all(exc) def handle_ChannelCloseOK(self, frame): """ AMQP server closed channel as per our request """ - # Let the close method continue working + assert self.channel._closing, "received a not expected CloseOk" + # Release the `close` method's future self.synchroniser.notify(spec.ChannelCloseOK) - exc = ClientChannelClosed() - self._close_all(exc) - def handle_ConnectionClose(self, frame): - """ AMQP server closed the connection with an error """ - # Make sure all `close` calls don't deadlock - self.channel._closing = True - exc = ServerConnectionClosed() - self._close_all(exc) - - def handle_ConnectionCloseOK(self, frame): - """ AMQP server closed connection as per our request """ - # Make sure all `close` calls don't deadlock - self.channel._closing = True - exc = ClientConnectionClosed() + exc = ClientChannelClosed() self._close_all(exc) def _close_all(self, exc): + # Make sure all `close` calls don't deadlock + self.channel._closed = True # If there were anyone who expected an `*-OK` kill them, as no data # will follow after close. Any new calls should also raise an error. self.synchroniser.killall(exc) - self.sender.killall(exc) # Cancel all consumers with same error self.consumers.error(exc) - # TODO: Add `handle_BasicCancel` for - # https://www.rabbitmq.com/consumer-cancel.html support - class MessageReceiver(object): def __init__(self, synchroniser, sender, consumers, reader): @@ -372,6 +375,7 @@ def receive_deliver(self, frame): payload.routing_key, payload.consumer_tag ) + # Delivers message to consumers when done self.is_getok_message = False self.reader.ready() diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 639465b..2f56119 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -2,6 +2,7 @@ import sys from . import spec from . import routing +from . import frames from .channel import ChannelFactory from .exceptions import ( AMQPConnectionError, ClientConnectionClosed, ServerConnectionClosed) @@ -35,7 +36,6 @@ class Connection(object): The :class:`~asyncio.Protocol` which is paired with the transport """ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): - self._loop = loop self.synchroniser = synchroniser self.sender = sender self.channel_factory = ChannelFactory(loop, protocol, dispatcher, connection_info) @@ -45,8 +45,7 @@ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, self.protocol = protocol # Indicates, that close was initiated by client self._closing = False - # List of created channels - self._close_waiters = [] + self._closed_with = None @asyncio.coroutine def open_channel(self): @@ -59,9 +58,15 @@ def open_channel(self): """ if self._closing: raise ClientConnectionClosed() + if self._closed_with: + raise self._closed_with + channel = yield from self.channel_factory.open() return channel + def is_closed(self): + return self._closing or self._closed_with is not None + @asyncio.coroutine def close(self): """ @@ -69,23 +74,23 @@ def close(self): This method is a :ref:`coroutine `. """ - if not self._closing: + if not self.is_closed(): self._closing = True # Let the ConnectionActor do the actual close operations. # It will do the work on CloseOK + self.sender.send_Close( + 0, 'Connection closed by application', 0, 0) try: - self.sender.send_Close( - 0, 'Connection closed by application', 0, 0) yield from self.synchroniser.await(spec.ConnectionCloseOK) except AMQPConnectionError: # For example if both sides want to close or the connection # is closed. pass else: - log.warn("Called `close` on already closed connection...") - # Wait for all components to close - if self._close_waiters: - yield from asyncio.gather(*self._close_waiters, loop=self._loop) + if self._closing: + log.warn("Called `close` on already closing connection...") + # finish all pending tasks + yield from self.protocol.heartbeat_monitor.wait_closed() @asyncio.coroutine @@ -94,7 +99,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): sender = ConnectionMethodSender(protocol) connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) - actor = ConnectionActor(synchroniser, sender, protocol, connection, loop=loop) + actor = ConnectionActor(synchroniser, sender, protocol, connection, dispatcher, loop=loop) reader = routing.QueuedReader(actor, loop=loop) try: @@ -132,10 +137,25 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): class ConnectionActor(routing.Actor): - def __init__(self, synchroniser, sender, protocol, connection, *, loop=None): + def __init__(self, synchroniser, sender, protocol, connection, dispatcher, *, loop=None): super().__init__(synchroniser, sender, loop=loop) self.protocol = protocol self.connection = connection + self.dispatcher = dispatcher + + def handle(self, frame): + # From docs on `close`: + # After sending this method, any received methods except Close and + # Close-OK MUST be discarded. + # So we will only process ConnectionClose, ConnectionCloseOK, + # PoisonPillFrame if channel is closed + if self.connection.is_closed(): + close_methods = (spec.ConnectionClose, spec.ConnectionCloseOK) + if isinstance(frame.payload, close_methods) or isinstance(frame, frames.PoisonPillFrame): + return super().handle(frame) + else: + return + return super().handle(frame) def handle_ConnectionStart(self, frame): self.synchroniser.notify(spec.ConnectionStart) @@ -146,26 +166,29 @@ def handle_ConnectionTune(self, frame): def handle_ConnectionOpenOK(self, frame): self.synchroniser.notify(spec.ConnectionOpenOK) + # Close handlers + def handle_PoisonPillFrame(self, frame): """ Is sent in case protocol lost connection to server.""" - # Make sure all `close` calls don't deadlock - self.connection._closing = True - exc = frame.exception - # Transport is already closed - self._close_all(exc) + # Will be delivered after Close or CloseOK handlers. It's for channels, + # so ignore it. + if self.connection._closed_with is not None: + return + # If connection was not closed already - we lost connection. + # Protocol should already be closed + self._close_all(frame.exception) def handle_ConnectionClose(self, frame): """ AMQP server closed the channel with an error """ # Notify server we are OK to close. self.sender.send_CloseOK() - self.connection._closing = True - exc = ServerConnectionClosed() - self._close_all(exc) - # Don't close transport right away, as CloseOK might not get to server - # yet. At least give the loop a spin before we do so. - # TODO: After FlowControl is implemented change this to drain and close - self._loop.call_soon(self.protocol.close) + exc = ServerConnectionClosed(frame.payload.reply_text, + frame.payload.reply_code) + self._close_all(exc) + # This will not abort transport, it will try to flush remaining data + # asynchronously, as stated in `asyncio` docs. + self.protocol.close() def handle_ConnectionCloseOK(self, frame): self.synchroniser.notify(spec.ConnectionCloseOK) @@ -175,14 +198,16 @@ def handle_ConnectionCloseOK(self, frame): self.protocol.close() def _close_all(self, exc): + # Make sure all `close` calls don't deadlock + self.connection._closed_with = exc # Close heartbeat self.protocol.heartbeat_monitor.stop() - self.connection._close_waiters.append( - self.protocol.heartbeat_monitor.wait_closed()) # If there were anyone who expected an `*-OK` kill them, as no data # will follow after close self.synchroniser.killall(exc) - self.sender.killall(exc) + # Notify all channels about error + poison_frame = frames.PoisonPillFrame(exc) + self.dispatcher.dispatch_all(poison_frame) class ConnectionMethodSender(routing.Sender): diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index e46a38e..5a40cd4 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -39,6 +39,11 @@ class ClientChannelClosed(AMQPChannelError): class ServerConnectionClosed(AMQPConnectionError): """ Connection was closed by server """ + def __init__(self, reply_text, reply_code): + super().__init__(reply_text) + self.reply_text = reply_text + self.reply_code = reply_code + class UndeliverableMessage(ValueError): pass diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 2067a2c..8ac26f2 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -8,7 +8,6 @@ class AMQP(asyncio.Protocol): def __init__(self, dispatcher, loop): - self._loop = loop self.dispatcher = dispatcher self.partial_frame = b'' self.frame_reader = FrameReader() diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index 1941ce2..1539bb8 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -257,13 +257,13 @@ def cancel(self): This method is a :ref:`coroutine `. """ + self.sender.send_BasicCancel(self.tag) try: - self.sender.send_BasicCancel(self.tag) yield from self.synchroniser.await(spec.BasicCancelOK) except AMQPError: pass else: - # No need to call ready if connection closed. + # No need to call ready if channel closed. self.reader.ready() self.cancelled = True self.cancelled_future.set_result(self) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index e4908c7..637f97b 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -1,7 +1,6 @@ import asyncio import collections from . import frames -from . import spec from .log import log @@ -18,15 +17,8 @@ def remove_handler(self, channel_id): def dispatch(self, frame): if isinstance(frame, frames.HeartbeatFrame): return - # XXX: Could not find a better way to close channels... - elif isinstance(frame.payload, (spec.ConnectionCloseOK, - spec.ConnectionClose)): - # Notify all connection channels about it - for handler in self.handlers.values(): - handler(frame) - else: - handler = self.handlers[frame.channel_id] - handler(frame) + handler = self.handlers[frame.channel_id] + handler(frame) def dispatch_all(self, frame): for handler in self.handlers.values(): @@ -37,19 +29,10 @@ class Sender(object): def __init__(self, channel_id, protocol): self.channel_id = channel_id self.protocol = protocol - self.connection_exc = None def send_method(self, method): - if self.connection_exc is not None: - raise self.connection_exc self.protocol.send_method(self.channel_id, method) - def killall(self, exc): - """ Connection/Channel was closed. All subsequent requests should - raise an error - """ - self._connection_exc = exc - class Actor(object): def __init__(self, synchroniser, sender, *, loop): diff --git a/test/channel_tests.py b/test/channel_tests.py index 2918ff0..ddb164c 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -240,7 +240,7 @@ def when_connection_is_closed(self): except Exception: pass self.tick() - self.was_closed = self.channel._closing + self.was_closed = self.channel.is_closed() def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) @@ -254,7 +254,7 @@ def when_connection_is_closed(self): self.task = asyncio.async(self.connection.close(), loop=self.loop) self.server.send_method(0, spec.ConnectionCloseOK()) self.tick() - self.was_closed = self.channel._closing + self.was_closed = self.channel.is_closed() self.loop.run_until_complete(asyncio.wait_for(self.task, 0.2)) def it_should_not_hang_channel_close(self): @@ -269,7 +269,7 @@ def when_connection_is_closed(self): self.server.send_method( 0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) self.tick() - self.was_closed = self.channel._closing + self.was_closed = self.channel.is_closed() def it_should_not_hang_channel_close(self): self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) @@ -291,7 +291,7 @@ def when_both_sides_close_channel(self): self.task.result() def if_should_have_closed_channel(self): - assert self.channel._closing + assert self.channel.is_closed() def it_should_have_killed_synchroniser_with_404(self): assert self.channel.synchroniser.connection_exc == exceptions.NotFound diff --git a/test/integration_tests.py b/test/integration_tests.py index b950837..6ec2cb3 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -304,13 +304,3 @@ def tear_down(): yield from self.connection.close() self.loop.run_until_complete(tear_down()) asyncio.set_event_loop(self.loop) - - -class WhenAConnectionClosedByHandshakeProtocolShouldNotDispatchPoisonPill(ConnectionContext): - def when_we_close_connection(self): - with mock.patch("asynqp.routing.Dispatcher.dispatch_all") as mocked: - self.loop.run_until_complete(self.connection.close()) - self.mocked = mocked - - def it_should_not_dispatch_poison(self): - assert not self.mocked.called From f4c2743fdf7cd9e44d272735b37323cf2318e5fd Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 3 Nov 2015 23:53:56 +0200 Subject: [PATCH 092/118] Renamed exceptions, got rid of ClientConnectionError and ServerConnectionError --- src/asynqp/channel.py | 7 +++---- src/asynqp/connection.py | 10 +++++----- src/asynqp/exceptions.py | 21 ++++++++------------- test/connection_tests.py | 5 +++-- test/integration_tests.py | 1 - 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 813601e..7c2c5a4 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -9,8 +9,7 @@ from . import message from . import routing from .exceptions import ( - UndeliverableMessage, ClientChannelClosed, - ServerConnectionClosed, ClientConnectionClosed, AMQPError) + UndeliverableMessage, AMQPError, ChannelClosed) from .log import log @@ -42,7 +41,7 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, self.queue_factory = queue_factory self.reader = reader self._closed = False - # Indicates, that channel is closed or started closing + # Indicates, that channel is closing by client(!) call self._closing = False @asyncio.coroutine @@ -330,7 +329,7 @@ def handle_ChannelCloseOK(self, frame): # Release the `close` method's future self.synchroniser.notify(spec.ChannelCloseOK) - exc = ClientChannelClosed() + exc = ChannelClosed() self._close_all(exc) def _close_all(self, exc): diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 2f56119..0384ef3 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -5,7 +5,7 @@ from . import frames from .channel import ChannelFactory from .exceptions import ( - AMQPConnectionError, ClientConnectionClosed, ServerConnectionClosed) + AMQPConnectionError, ConnectionClosed) from .log import log @@ -57,7 +57,7 @@ def open_channel(self): :return: The new :class:`Channel` object. """ if self._closing: - raise ClientConnectionClosed() + raise ConnectionClosed("Closed by application") if self._closed_with: raise self._closed_with @@ -183,8 +183,8 @@ def handle_ConnectionClose(self, frame): # Notify server we are OK to close. self.sender.send_CloseOK() - exc = ServerConnectionClosed(frame.payload.reply_text, - frame.payload.reply_code) + exc = ConnectionClosed(frame.payload.reply_text, + frame.payload.reply_code) self._close_all(exc) # This will not abort transport, it will try to flush remaining data # asynchronously, as stated in `asyncio` docs. @@ -192,7 +192,7 @@ def handle_ConnectionClose(self, frame): def handle_ConnectionCloseOK(self, frame): self.synchroniser.notify(spec.ConnectionCloseOK) - exc = ClientConnectionClosed() + exc = ConnectionClosed("Closed by application") self._close_all(exc) # We already agread with server on closing, so lets do it right away self.protocol.close() diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 5a40cd4..22ad48b 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -5,9 +5,8 @@ __all__ = [ "AMQPError", "ConnectionLostError", - "ServerConnectionClosed", - "ClientChannelClosed", - "ClientConnectionClosed", + "ChannelClosed", + "ConnectionClosed", "AMQPChannelError", "AMQPConnectionError", "UndeliverableMessage", @@ -28,23 +27,19 @@ def __init__(self, message, exc=None): self.original_exc = exc -class ClientConnectionClosed(AMQPConnectionError): +class ConnectionClosed(AMQPConnectionError): """ Connection was closed by client """ - -class ClientChannelClosed(AMQPChannelError): - """ Channel was closed by client """ - - -class ServerConnectionClosed(AMQPConnectionError): - """ Connection was closed by server """ - - def __init__(self, reply_text, reply_code): + def __init__(self, reply_text, reply_code=None): super().__init__(reply_text) self.reply_text = reply_text self.reply_code = reply_code +class ChannelClosed(AMQPChannelError): + """ Channel was closed by client """ + + class UndeliverableMessage(ValueError): pass diff --git a/test/connection_tests.py b/test/connection_tests.py index ddd60f6..3b09930 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -147,7 +147,7 @@ def when_client_closes_connection(self): def it_should_raise_error_in_connection_methods(self): exc = contexts.catch(self.wait_for, self.connection.open_channel()) - assert isinstance(exc, exceptions.ClientConnectionClosed) + assert isinstance(exc, exceptions.ConnectionClosed) class WhenServerAndClientCloseConnectionAtATime(OpenConnectionContext): @@ -168,4 +168,5 @@ def if_should_have_closed_connection(self): def it_should_have_killed_synchroniser_with_server_error(self): assert isinstance( self.connection.synchroniser.connection_exc, - exceptions.ServerConnectionClosed) + exceptions.ConnectionClosed) + assert self.connection.synchroniser.connection_exc.reply_code == 123 diff --git a/test/integration_tests.py b/test/integration_tests.py index 6ec2cb3..6294576 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -3,7 +3,6 @@ import socket import contexts from asyncio import test_utils -from unittest import mock class ConnectionContext: From 7df87b792dec3cdd52806966093ec7b43258d5af Mon Sep 17 00:00:00 2001 From: Taras Date: Wed, 2 Dec 2015 18:38:28 +0200 Subject: [PATCH 093/118] Added nowait parameters to queue and exchange declarations --- src/asynqp/channel.py | 26 ++++++++++++++++---------- src/asynqp/queue.py | 12 ++++++++---- test/exchange_tests.py | 20 ++++++++++++++++++++ test/queue_tests.py | 18 ++++++++++++++++++ 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index f92174b..1c51e64 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -46,7 +46,8 @@ def __init__(self, id, synchroniser, sender, basic_return_consumer, @asyncio.coroutine def declare_exchange(self, name, type, *, durable=True, auto_delete=False, - passive=False, internal=False, arguments=None): + passive=False, internal=False, nowait=False, + arguments=None): """ Declare an :class:`Exchange` on the broker. If the exchange does not exist, it will be created. @@ -65,6 +66,8 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, ``internal`` are ignored if `passive=True`. :keyword bool internal: If true, the exchange cannot be published to directly; it can only be bound to other exchanges. + :keyword bool nowait: If true, the method will not wait for declare-ok + to arrive and return right away. :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. @@ -80,18 +83,20 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, "period, or colon, and do not begin with 'amq.'") self.sender.send_ExchangeDeclare( - name, type, passive, durable, auto_delete, internal, + name, type, passive, durable, auto_delete, internal, nowait, arguments or {}) - yield from self.synchroniser.await(spec.ExchangeDeclareOK) + if not nowait: + yield from self.synchroniser.await(spec.ExchangeDeclareOK) + self.reader.ready() ex = exchange.Exchange( self.reader, self.synchroniser, self.sender, name, type, durable, auto_delete, internal) - self.reader.ready() return ex @asyncio.coroutine def declare_queue(self, name='', *, durable=True, exclusive=False, - auto_delete=False, passive=False, arguments=None): + auto_delete=False, passive=False, + nowait=False, arguments=None): """ Declare a queue on the broker. If the queue does not exist, it will be created. @@ -108,12 +113,13 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, exist it will raise a :class:`exceptions.NotFound` instead of creating it. Arguments ``durable``, ``auto_delete`` and ``exclusive`` are ignored if ``passive=True``. + :keyword bool nowait: If true, will not wait for a declare-ok to arrive. :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`Queue` object. """ q = yield from self.queue_factory.declare( - name, durable, exclusive, auto_delete, passive, + name, durable, exclusive, auto_delete, passive, nowait, arguments if arguments is not None else {}) return q @@ -446,14 +452,14 @@ def __init__(self, channel_id, protocol, connection_info): def send_ChannelOpen(self): self.send_method(spec.ChannelOpen('')) - def send_ExchangeDeclare(self, name, type, passive, durable, auto_delete, internal, arguments): - self.send_method(spec.ExchangeDeclare(0, name, type, passive, durable, auto_delete, internal, False, arguments)) + def send_ExchangeDeclare(self, name, type, passive, durable, auto_delete, internal, nowait, arguments): + self.send_method(spec.ExchangeDeclare(0, name, type, passive, durable, auto_delete, internal, nowait, arguments)) def send_ExchangeDelete(self, name, if_unused): self.send_method(spec.ExchangeDelete(0, name, if_unused, False)) - def send_QueueDeclare(self, name, durable, exclusive, auto_delete, passive, arguments): - self.send_method(spec.QueueDeclare(0, name, passive, durable, exclusive, auto_delete, False, arguments)) + def send_QueueDeclare(self, name, durable, exclusive, auto_delete, passive, nowait, arguments): + self.send_method(spec.QueueDeclare(0, name, passive, durable, exclusive, auto_delete, nowait, arguments)) def send_QueueBind(self, queue_name, exchange_name, routing_key, arguments): self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, arguments)) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index 29996da..61b50a0 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -280,21 +280,25 @@ def __init__(self, sender, synchroniser, reader, consumers, *, loop): self.consumers = consumers @asyncio.coroutine - def declare(self, name, durable, exclusive, auto_delete, passive, + def declare(self, name, durable, exclusive, auto_delete, passive, nowait, arguments): if not VALID_QUEUE_NAME_RE.match(name): raise ValueError( "Not a valid queue name.\n" "Valid names consist of letters, digits, hyphen, underscore, " "period, or colon, and do not begin with 'amq.'") + if not name and nowait: + raise ValueError("Declaring the queue without `name` and with " + "`nowait` is forbidden") self.sender.send_QueueDeclare( - name, durable, exclusive, auto_delete, passive, arguments) - name = yield from self.synchroniser.await(spec.QueueDeclareOK) + name, durable, exclusive, auto_delete, passive, nowait, arguments) + if not nowait: + name = yield from self.synchroniser.await(spec.QueueDeclareOK) + self.reader.ready() q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete, arguments, loop=self._loop) - self.reader.ready() return q diff --git a/test/exchange_tests.py b/test/exchange_tests.py index ff7161b..ac302e3 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -213,3 +213,23 @@ def when_error_arrives(self): def it_should_raise_not_found_error(self): assert isinstance(self.task.exception(), exceptions.NotFound) + + +class WhenIDeclareExchangeWithNoWait(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.async(self.channel.declare_exchange( + 'my.nice.exchange', 'fanout', durable=True, auto_delete=False, + internal=False, nowait=True), loop=self.loop) + self.tick() + + def it_should_return_exchange_object_without_wait(self): + result = self.task.result() + assert result + assert result.name == 'my.nice.exchange' + assert result.type == 'fanout' + + def it_should_have_sent_nowait_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.ExchangeDeclare( + 0, 'my.nice.exchange', 'fanout', False, True, False, False, + True, {})) diff --git a/test/queue_tests.py b/test/queue_tests.py index f7ccdce..4025135 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -450,3 +450,21 @@ def when_error_arrives(self): def it_should_raise_exception(self): assert isinstance(self.task.exception(), exceptions.NotFound) + + +class WhenIDeclareQueueWithNoWait(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.async(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=False, + nowait=True), loop=self.loop) + self.tick() + + def it_should_return_queue_object_without_wait(self): + result = self.task.result() + assert result + assert result.name == '123' + + def it_should_have_sent_nowait_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.QueueDeclare( + 0, '123', False, True, True, False, True, {})) From 8f55f2f03cba8aea7c8f1532fa283df3bc50b02e Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Fri, 11 Mar 2016 14:51:49 +0000 Subject: [PATCH 094/118] ignore pep8 W503 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 39956d0..86996fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: - python setup.py develop script: - - flake8 src test --ignore=E501 + - flake8 src test --ignore=E501,W503 - coverage run --source=src -m contexts -v - pushd doc && make html && popd From cab97e9233dbc8cce90054fe1f6188d2bda23899 Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 1 Nov 2015 22:34:19 +0200 Subject: [PATCH 095/118] Added server side consumer cancel https://www.rabbitmq.com/consumer-cancel.html --- src/asynqp/channel.py | 7 +++++++ src/asynqp/connection.py | 5 ++++- src/asynqp/queue.py | 8 +++++++- test/connection_tests.py | 5 ++++- test/queue_tests.py | 16 ++++++++++++++++ 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 1c51e64..711a564 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -227,6 +227,7 @@ def open(self): actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) actor.consumers = consumers actor.channel = channel + actor.reader = reader self.dispatcher.add_handler(channel_id, reader.feed) try: @@ -255,6 +256,7 @@ def __init__(self, *args, **kwargs): self.channel = None self.consumers = None self.message_receiver = None + self.reader = None def handle(self, frame): # From docs on `close`: @@ -307,6 +309,11 @@ def handle_BasicCancelOK(self, frame): def handle_BasicQosOK(self, frame): self.synchroniser.notify(spec.BasicQosOK) + def handle_BasicCancel(self, frame): + # Cancel consumer server-side + self.consumers.server_cancel(frame.payload.consumer_tag) + self.reader.ready() + # Message receiving hanlers def handle_BasicGetOK(self, frame): diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 0384ef3..3fe0716 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -111,7 +111,10 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): sender.send_StartOK( {"product": "asynqp", "version": "0.1", # todo: use pkg_resources to inspect the package - "platform": sys.version}, + "platform": sys.version, + "capabilities": { + "consumer_cancel_notify": True + }}, 'AMQPLAIN', {'LOGIN': connection_info['username'], 'PASSWORD': connection_info['password']}, 'en_US' diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index 61b50a0..af538f6 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -310,13 +310,19 @@ def __init__(self, loop): def add_consumer(self, consumer): self.consumers[consumer.tag] = consumer # so the consumer gets garbage collected when it is cancelled - consumer.cancelled_future.add_done_callback(lambda fut: delitem(self.consumers, fut.result().tag)) + consumer.cancelled_future.add_done_callback(lambda fut: self.consumers.pop(fut.result().tag, None)) def deliver(self, tag, msg): assert tag in self.consumers, "Message got delivered to a non existent consumer" consumer = self.consumers[tag] self.loop.call_soon(consumer.callback, msg) + def server_cancel(self, tag): + consumer = self.consumers.pop(tag, None) + if consumer: + if hasattr(consumer.callback, 'on_cancel'): + consumer.callback.on_cancel() + def error(self, exc): for consumer in self.consumers.values(): if hasattr(consumer.callback, 'on_error'): diff --git a/test/connection_tests.py b/test/connection_tests.py index 3b09930..080c215 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -16,7 +16,10 @@ def when_ConnectionStart_arrives(self): def it_should_send_start_ok(self): expected_method = spec.ConnectionStartOK( - {"product": "asynqp", "version": "0.1", "platform": sys.version}, + {"product": "asynqp", "version": "0.1", "platform": sys.version, + "capabilities": { + "consumer_cancel_notify": True + }}, 'AMQPLAIN', {'LOGIN': 'guest', 'PASSWORD': 'guest'}, 'en_US' diff --git a/test/queue_tests.py b/test/queue_tests.py index 4025135..411b6fa 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -468,3 +468,19 @@ def it_should_have_sent_nowait_in_frame(self): self.server.should_have_received_method( self.channel.id, spec.QueueDeclare( 0, '123', False, True, True, False, True, {})) + + +class WhenConsumerIsClosedServerSide(QueueContext, ExchangeContext): + def given_a_consumer(self): + task = asyncio.async(self.queue.consume(lambda x: None)) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.consumer = task.result() + + def when_consumer_is_closed_server_side(self): + self.server.send_method(self.channel.id, spec.BasicCancel('made.up.tag', False)) + self.tick() + + def it_should_close_consumer(self): + assert 'made.up.tag' not in self.channel.queue_factory.consumers.consumers From 036e10711a279c213fdc306e6923dddc0932559b Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 1 Nov 2015 22:47:08 +0200 Subject: [PATCH 096/118] Fix flake errors --- src/asynqp/connection.py | 2 +- src/asynqp/queue.py | 1 - test/connection_tests.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 3fe0716..2d5332b 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -113,7 +113,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): "version": "0.1", # todo: use pkg_resources to inspect the package "platform": sys.version, "capabilities": { - "consumer_cancel_notify": True + "consumer_cancel_notify": True }}, 'AMQPLAIN', {'LOGIN': connection_info['username'], 'PASSWORD': connection_info['password']}, diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index af538f6..e561773 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -1,6 +1,5 @@ import asyncio import re -from operator import delitem from . import spec from .exceptions import Deleted, AMQPError diff --git a/test/connection_tests.py b/test/connection_tests.py index 080c215..c0c76e2 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -18,7 +18,7 @@ def it_should_send_start_ok(self): expected_method = spec.ConnectionStartOK( {"product": "asynqp", "version": "0.1", "platform": sys.version, "capabilities": { - "consumer_cancel_notify": True + "consumer_cancel_notify": True }}, 'AMQPLAIN', {'LOGIN': 'guest', 'PASSWORD': 'guest'}, From 13b7c1133aa18e75dea204c757cc583b73e8df23 Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 7 Dec 2015 16:25:05 +0200 Subject: [PATCH 097/118] Fixed amqp properties reading. Had invalid flag checks. --- src/asynqp/message.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/asynqp/message.py b/src/asynqp/message.py index 95a0b74..426b6f6 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -208,8 +208,10 @@ def read(cls, raw): property_flags_short = serialisation.read_unsigned_short(bytesio) properties = [] - for amqptype, flag in zip(Message.property_types.values(), bin(property_flags_short)[2:]): - if flag == '1': + + for i, amqptype in enumerate(Message.property_types.values()): + pos = 15 - i # We started from `content_type` witch has pos==15 + if property_flags_short & (1 << pos): properties.append(amqptype.read(bytesio)) else: properties.append(None) From 5bc9ce3d8ad409835cc91fce9137d7af3974a8df Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 8 Mar 2016 16:13:45 +0200 Subject: [PATCH 098/118] Added test for reading content properties with some missing bits --- src/asynqp/message.py | 4 ++++ test/message_tests.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/asynqp/message.py b/src/asynqp/message.py index 426b6f6..9278c97 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -218,6 +218,10 @@ def read(cls, raw): return cls(class_id, body_length, properties) + def __repr__(self): + return "".format( + self.class_id, self.body_length, self.properties) + class MessageBuilder(object): def __init__(self, sender, delivery_tag, redelivered, exchange_name, routing_key, consumer_tag=None): diff --git a/test/message_tests.py b/test/message_tests.py index 21d914d..ab61a22 100644 --- a/test/message_tests.py +++ b/test/message_tests.py @@ -220,3 +220,19 @@ def when_I_set_a_property(self): def it_should_not_attempt_to_cast_it(self): assert self.msg.foo == 123 + + +class WhenIReadAContentHeaderWithoutAllProperties: + + def given_headers_wit_only_content_encoding(self): + self.data = ( + b'\x00<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08p\x00' + b'\x05utf-8\x00\x00\x00\x00\x01') + + def when_I_read_properties(self): + self.payload = message.ContentHeaderPayload.read(self.data) + + def it_should_have_only_content_encoding(self): + assert self.payload == message.ContentHeaderPayload( + 60, 8, [None, 'utf-8', {}, 1, None, None, + None, None, None, None, None, None, None]) From 142d98ec1a0f0c0f8c360aa5186693ff6058a87d Mon Sep 17 00:00:00 2001 From: Taras Date: Tue, 8 Mar 2016 16:45:13 +0200 Subject: [PATCH 099/118] Tiny timing increase so tests pass on travis --- test/integration_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration_tests.py b/test/integration_tests.py index 6294576..67d1469 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -24,10 +24,10 @@ def cleanup_the_channel(self): class BoundQueueContext(ChannelContext): def given_a_queue_bound_to_an_exchange(self): - self.loop.run_until_complete(asyncio.wait_for(self.setup(), 0.4)) + self.loop.run_until_complete(asyncio.wait_for(self.setup(), 0.5)) def cleanup_the_queue_and_exchange(self): - self.loop.run_until_complete(asyncio.wait_for(self.teardown(), 0.2)) + self.loop.run_until_complete(asyncio.wait_for(self.teardown(), 0.3)) @asyncio.coroutine def setup(self): From 45e8619dce3639ac22018cff8697c1c033ca347c Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 20 Mar 2016 16:38:51 +0200 Subject: [PATCH 100/118] Added v0.5 changes and fixed conformance doc page --- CHANGELOG.md | 16 ++++++++++++++++ doc/conformance.rst | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a39b2..dceb3b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,22 @@ What's new in `asynqp` ====================== + +v0.5 +---- +* Channels will no longer break if their calls are cancelled. Issue #52. +* Fixed message properties decoding without `content_type`. Pull request #66. +* Added `nowait` argument to Exchange and Queue declarations. +* Added `passive` argument to Exchange and Queue declarations. +* Added `on_error` and `on_cancel` callbacks for Consumer. Issue #34 +* Changed the closing scheme for Channel/Connection. Proper exceptions are now +always propagated to user. Issues #57, #58 +* Complete internals refactor and cleanup. Rull requests #48, #49, #50. +* Add NO_DELAY option for socket. Issue #40. (Thanks to @socketpair for PR #41) +* Change heartbeat to be a proper background task. Issue #45. +* `loop` is now proparly passed to all components from open_connection call. Pull request #42. +* Add support for Python up to 3.5. + v0.4 ---- diff --git a/doc/conformance.rst b/doc/conformance.rst index 3ebfd9e..70fe83b 100644 --- a/doc/conformance.rst +++ b/doc/conformance.rst @@ -46,7 +46,7 @@ that are currently supported by ``asynqp``. +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | exchange | | :orange:`partial` | :class:`asynqp.Exchange` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ -| | declare/declare-ok | :orange:`partial` | :meth:`asynqp.Channel.declare_exchange` | Not all parameters presently supported | +| | declare/declare-ok | :green:`full` | :meth:`asynqp.Channel.declare_exchange` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | | delete/delete-ok | :green:`full` | :meth:`asynqp.Exchange.delete` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ @@ -56,7 +56,7 @@ that are currently supported by ``asynqp``. +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | queue | | :orange:`partial` | :class:`asynqp.Queue` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ -| | declare/declare-ok | :orange:`partial` | :meth:`asynqp.Channel.declare_queue` | Not all parameters presently supported | +| | declare/declare-ok | :green:`full` | :meth:`asynqp.Channel.declare_queue` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | | bind/bind-ok | :orange:`partial` | :meth:`asynqp.Queue.bind` | Not all parameters presently supported | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ From f824740f68b9bcc918c95442a52f18ede4423623 Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 20 Mar 2016 17:29:13 +0200 Subject: [PATCH 101/118] Added backward compatible closed future. --- src/asynqp/connection.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 0384ef3..1c5a0da 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -23,10 +23,6 @@ class Connection(object): Connections are created using :func:`asynqp.connect() `. - .. attribute:: closed - - a :class:`~asyncio.Future` which is done when the handshake to close the connection has finished - .. attribute:: transport The :class:`~asyncio.BaseTransport` over which the connection is communicating with the server @@ -44,8 +40,11 @@ def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, self.transport = transport self.protocol = protocol # Indicates, that close was initiated by client + self.closed = asyncio.Future(loop=loop) + # This future is a backport, so we don't need to log pending errors + self.closed.add_done_callback(lambda fut: fut.exception()) + self._closing = False - self._closed_with = None @asyncio.coroutine def open_channel(self): @@ -58,14 +57,15 @@ def open_channel(self): """ if self._closing: raise ConnectionClosed("Closed by application") - if self._closed_with: - raise self._closed_with + if self.closed.done(): + raise self.closed.exception() channel = yield from self.channel_factory.open() return channel def is_closed(self): - return self._closing or self._closed_with is not None + " Returns True if connection was closed " + return self._closing or self.closed.done() @asyncio.coroutine def close(self): @@ -172,7 +172,7 @@ def handle_PoisonPillFrame(self, frame): """ Is sent in case protocol lost connection to server.""" # Will be delivered after Close or CloseOK handlers. It's for channels, # so ignore it. - if self.connection._closed_with is not None: + if self.connection.closed.done(): return # If connection was not closed already - we lost connection. # Protocol should already be closed @@ -199,7 +199,7 @@ def handle_ConnectionCloseOK(self, frame): def _close_all(self, exc): # Make sure all `close` calls don't deadlock - self.connection._closed_with = exc + self.connection.closed.set_exception(exc) # Close heartbeat self.protocol.heartbeat_monitor.stop() # If there were anyone who expected an `*-OK` kill them, as no data From ec3122026dee0473b524ac064f968b029de15978 Mon Sep 17 00:00:00 2001 From: Taras Date: Sun, 20 Mar 2016 18:33:35 +0200 Subject: [PATCH 102/118] Changed the reconnect example to show new features in 0.5 --- doc/examples/reconnecting.py | 204 ++++++++++++----------------------- 1 file changed, 71 insertions(+), 133 deletions(-) diff --git a/doc/examples/reconnecting.py b/doc/examples/reconnecting.py index 78831b0..577506d 100644 --- a/doc/examples/reconnecting.py +++ b/doc/examples/reconnecting.py @@ -1,154 +1,92 @@ -''' -Example async consumer and publisher that will reconnect -automatically when a connection to rabbitmq is broken and -restored. -Note that no attempt is made to re-send messages that are -generated while the connection is down. -''' import asyncio import asynqp -from asyncio.futures import InvalidStateError +import logging +logging.basicConfig(level=logging.INFO) -# Global variables are ugly, but this is a simple example -CHANNELS = [] -CONNECTION = None -CONSUMER = None -PRODUCER = None +RECONNECT_BACKOFF = 1 -@asyncio.coroutine -def setup_connection(loop): - # connect to the RabbitMQ broker - connection = yield from asynqp.connect('localhost', - 5672, - username='guest', - password='guest') - return connection - +class Consumer: -@asyncio.coroutine -def setup_exchange_and_queue(connection): - # Open a communications channel - channel = yield from connection.open_channel() - - # Create a queue and an exchange on the broker - exchange = yield from channel.declare_exchange('test.exchange', 'direct') - queue = yield from channel.declare_queue('test.queue') + def __init__(self, connection, queue): + self.queue = queue + self.connection = connection - # Save a reference to each channel so we can close it later - CHANNELS.append(channel) + def __call__(self, msg): + self.queue.put_nowait(msg) - # Bind the queue to the exchange, so the queue will get messages published to the exchange - yield from queue.bind(exchange, 'routing.key') - - return exchange, queue + def on_error(self, exc): + print("Connection lost while consuming queue", exc) @asyncio.coroutine -def setup_consumer(connection): - # callback will be called each time a message is received from the queue - def callback(msg): - print('Received: {}'.format(msg.body)) - msg.ack() - - _, queue = yield from setup_exchange_and_queue(connection) - - # connect the callback to the queue - consumer = yield from queue.consume(callback) - return consumer +def connect_and_consume(queue): + # connect to the RabbitMQ broker + connection = yield from asynqp.connect( + 'localhost', 5672, username='guest', password='guest') + try: + channel = yield from connection.open_channel() + amqp_queue = yield from channel.declare_queue('test.queue') + consumer = Consumer(connection, queue) + yield from amqp_queue.consume(consumer) + except asynqp.AMQPError as err: + print("Could not consume on queue", err) + yield from connection.close() + return None + return connection @asyncio.coroutine -def setup_producer(connection): - ''' - The producer will live as an asyncio.Task - to stop it call Task.cancel() - ''' - exchange, _ = yield from setup_exchange_and_queue(connection) - - count = 0 - while True: - msg = asynqp.Message('Message #{}'.format(count)) - exchange.publish(msg, 'routing.key') - yield from asyncio.sleep(1) - count += 1 +def reconnector(queue): + try: + connection = None + while True: + if connection is None or connection.is_closed(): + print("Connecting to rabbitmq...") + try: + connection = yield from connect_and_consume(queue) + except (ConnectionError, OSError): + print("Failed to connect to rabbitmq server. " + "Will retry in {} seconds".format(RECONNECT_BACKOFF)) + connection = None + if connection is None: + yield from asyncio.sleep(RECONNECT_BACKOFF) + else: + print("Successfully connected and consuming test.queue") + # poll connection state every 100ms + yield from asyncio.sleep(0.1) + except asyncio.CancelledError: + if connection is not None: + yield from connection.close() @asyncio.coroutine -def start(loop): - ''' - Creates a connection, starts the consumer and producer. - If it fails, it will attempt to reconnect after waiting - 1 second - ''' - global CONNECTION - global CONSUMER - global PRODUCER +def process_msgs(queue): try: - CONNECTION = yield from setup_connection(loop) - CONSUMER = yield from setup_consumer(CONNECTION) - PRODUCER = loop.create_task(setup_producer(CONNECTION)) - # Multiple exceptions may be thrown, ConnectionError, OsError - except Exception: - print('failed to connect, trying again.') - yield from asyncio.sleep(1) - loop.create_task(start(loop)) + while True: + msg = yield from queue.get() + print("Received", msg.body) + msg.ack() + except asyncio.CancelledError: + pass + + +def main(): + loop = asyncio.get_event_loop() + queue = asyncio.Queue() + # Start main indexing task in the background + reconnect_task = loop.create_task(reconnector(queue)) + process_task = loop.create_task(process_msgs(queue)) + try: + loop.run_forever() + except KeyboardInterrupt: + process_task.cancel() + reconnect_task.cancel() + loop.run_until_complete(process_task) + loop.run_until_complete(reconnect_task) + loop.close() -@asyncio.coroutine -def stop(): - ''' - Cleans up connections, channels, consumers and producers - when the connection is closed. - ''' - global CHANNELS - global CONNECTION - global PRODUCER - global CONSUMER - - yield from CONSUMER.cancel() # this is a coroutine - PRODUCER.cancel() # this is not - - for channel in CHANNELS: - yield from channel.close() - CHANNELS = [] - - if CONNECTION is not None: - try: - yield from CONNECTION.close() - except InvalidStateError: - pass # could be automatically closed, so this is expected - CONNECTION = None - - -def connection_lost_handler(loop, context): - ''' - Here we setup a custom exception handler to listen for - ConnectionErrors. - - The exceptions we can catch follow this inheritance scheme - - - ConnectionError - base - | - - asynqp.exceptions.ConnectionClosedError - connection closed properly - | - - asynqp.exceptions.ConnectionLostError - closed unexpectedly - ''' - exception = context.get('exception') - if isinstance(exception, asynqp.exceptions.ConnectionClosedError): - print('Connection lost -- trying to reconnect') - # close everything before recpnnecting - close_task = loop.create_task(stop()) - asyncio.wait_for(close_task, None) - # reconnect - loop.create_task(start(loop)) - else: - # default behaviour - loop.default_exception_handler(context) - - -loop = asyncio.get_event_loop() -loop.set_exception_handler(connection_lost_handler) -loop.create_task(start(loop)) -loop.run_forever() +if __name__ == "__main__": + main() From c0b1ea1f076509edc4f59800100b1d55542801b3 Mon Sep 17 00:00:00 2001 From: Anton Ryzhov Date: Sun, 20 Mar 2016 21:32:10 +0300 Subject: [PATCH 103/118] Correctly handle zero-length messages --- src/asynqp/channel.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 1c51e64..bcb5bd3 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -422,27 +422,34 @@ def receive_return(self, frame): def receive_header(self, frame): assert self.message_builder is not None, "Received unexpected header" self.message_builder.set_header(frame.payload) + if self.message_builder.done(): + self.message_done() + return self.reader.ready() def receive_body(self, frame): assert self.message_builder is not None, "Received unexpected body" self.message_builder.add_body_chunk(frame.payload) - if self.message_builder.done(): - msg = self.message_builder.build() - tag = self.message_builder.consumer_tag - if self.is_getok_message: - self.synchroniser.notify(spec.BasicGetOK, (tag, msg)) - # Dont call ready() if message arrive after GetOk. It's the - # ``Queue.get`` method's responsibility - else: - self.consumers.deliver(tag, msg) - self.reader.ready() - self.message_builder = None + if self.message_builder.done(): + self.message_done() return # If message is not done yet we still need more frames. Wait for them self.reader.ready() + def message_done(self): + msg = self.message_builder.build() + tag = self.message_builder.consumer_tag + if self.is_getok_message: + self.synchroniser.notify(spec.BasicGetOK, (tag, msg)) + # Dont call ready() if message arrive after GetOk. It's the + # ``Queue.get`` method's responsibility + else: + self.consumers.deliver(tag, msg) + self.reader.ready() + + self.message_builder = None + class ChannelMethodSender(routing.Sender): def __init__(self, channel_id, protocol, connection_info): From b77ad904153198db4fbc297156f681da3277a9ab Mon Sep 17 00:00:00 2001 From: Taras Date: Mon, 21 Mar 2016 10:13:11 +0300 Subject: [PATCH 104/118] Unittest --- test/integration_tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/integration_tests.py b/test/integration_tests.py index 6294576..5c13c83 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -303,3 +303,21 @@ def tear_down(): yield from self.connection.close() self.loop.run_until_complete(tear_down()) asyncio.set_event_loop(self.loop) + + +class WhenISendZeroMessage(BoundQueueContext): + def given_an_empty_message(self): + self.message = asynqp.Message('') + self.exchange.publish(self.message, 'routingkey') + + def when_I_start_a_consumer(self): + self.message_received = asyncio.Future() + self.loop.run_until_complete(asyncio.wait_for(self.start_consumer(), 0.2)) + + def it_should_deliver_the_message_to_the_consumer(self): + assert self.message_received.result() == self.message + + @asyncio.coroutine + def start_consumer(self): + yield from self.queue.consume(self.message_received.set_result) + yield from self.message_received From 6204715e1bf86eb0a97a41a0dd5b292f2808cef5 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 29 Jun 2016 16:23:17 -0400 Subject: [PATCH 105/118] Bump version to 0.5 --- doc/conf.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index a32388b..dd0f550 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '0.4' +version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.4' +release = '0.5' def hide_class_constructor(app, what, name, obj, options, signature, return_annotation): diff --git a/setup.py b/setup.py index 461da0e..936725e 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='asynqp', - version='0.4', + version='0.5', author="Benjamin Hodgson", author_email="benjamin.hodgson@huddle.net", url="https://github.com/benjamin-hodgson/asynqp", From 8e298e12a8f0615e218f40314423a7c76204b588 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 29 Jun 2016 16:39:07 -0400 Subject: [PATCH 106/118] >:( flake8 bug --- src/asynqp/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index 9b101b5..d457d88 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,3 +1,5 @@ +# flake8: noqa + import socket import asyncio from .exceptions import * # noqa From fddabe89d084a634650e2c7ffc3413a2e65e11be Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 29 Jun 2016 16:39:41 -0400 Subject: [PATCH 107/118] bump again --- doc/conf.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index dd0f550..c966a22 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '0.5' +version = '0.5.1' # The full version, including alpha/beta/rc tags. -release = '0.5' +release = '0.5.1' def hide_class_constructor(app, what, name, obj, options, signature, return_annotation): diff --git a/setup.py b/setup.py index 936725e..3ff694c 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='asynqp', - version='0.5', + version='0.5.1', author="Benjamin Hodgson", author_email="benjamin.hodgson@huddle.net", url="https://github.com/benjamin-hodgson/asynqp", From 59e839f81f4e230370a5a13b4e2c00b07ce73d08 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Wed, 29 Jun 2016 17:02:36 -0400 Subject: [PATCH 108/118] Throw when binding to an invalid exchange --- src/asynqp/exceptions.py | 4 ++++ src/asynqp/queue.py | 4 +++- test/queue_tests.py | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 22ad48b..4f94862 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -48,6 +48,10 @@ class Deleted(ValueError): pass +class InvalidExchangeName(ValueError): + pass + + globals().update(EXCEPTIONS) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index e561773..eac96ef 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -1,7 +1,7 @@ import asyncio import re from . import spec -from .exceptions import Deleted, AMQPError +from .exceptions import Deleted, AMQPError, InvalidExchangeName VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -68,6 +68,8 @@ def bind(self, exchange, routing_key, *, arguments=None): """ if self.deleted: raise Deleted("Queue {} was deleted".format(self.name)) + if not exchange: + raise InvalidExchangeName("Can't bind queue {} to the default exchange".format(self.name)) self.sender.send_QueueBind(self.name, exchange.name, routing_key, arguments or {}) yield from self.synchroniser.await(spec.QueueBindOK) diff --git a/test/queue_tests.py b/test/queue_tests.py index 411b6fa..0bac897 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -81,6 +81,14 @@ def it_should_send_QueueBind(self): self.server.should_have_received_method(self.channel.id, expected_method) +class WhenBindingAQueueToTheDefaultExchange(QueueContext): + def when_I_bind_the_queue(self): + self.task = self.async_partial(self.queue.bind('', 'routing.key', arguments={'x-ignore': ''})) + + def it_should_throw_InvalidExchangeName(self): + assert isinstance(self.task.exception(), exceptions.InvalidExchangeName) + + class WhenQueueBindOKArrives(QueueContext, ExchangeContext): def given_I_sent_QueueBind(self): self.task = asyncio.async(self.queue.bind(self.exchange, 'routing.key')) From 9d1b30001167bef9f6ec5d890e449a750854548f Mon Sep 17 00:00:00 2001 From: purpleP Date: Fri, 12 Aug 2016 03:21:35 +0300 Subject: [PATCH 109/118] Changed all occurences of 'try except pass' into 'with suppress' context manager (#81) * changed all "try except pass" occurences to suppress context manager * typo fix * removed unused import --- src/asynqp/protocol.py | 9 +++------ test/channel_tests.py | 5 ++--- test/connection_tests.py | 9 +++------ test/queue_tests.py | 9 +++------ 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 8ac26f2..fa1157b 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -1,5 +1,6 @@ import asyncio import struct +from contextlib import suppress from . import spec from . import frames from .exceptions import AMQPError, ConnectionLostError @@ -127,15 +128,11 @@ def stop(self): @asyncio.coroutine def wait_closed(self): if self.send_hb_task is not None: - try: + with suppress(asyncio.CancelledError): yield from self.send_hb_task - except asyncio.CancelledError: - pass if self.monitor_task is not None: - try: + with suppress(asyncio.CancelledError): yield from self.monitor_task - except asyncio.CancelledError: - pass @asyncio.coroutine def send_heartbeat(self, interval): diff --git a/test/channel_tests.py b/test/channel_tests.py index ddb164c..8e20532 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -1,6 +1,7 @@ import asyncio import contexts import asynqp +from contextlib import suppress from unittest import mock from asynqp import spec, frames, exceptions from asynqp import message @@ -235,10 +236,8 @@ def it_should_throw_a_TypeError(self): class WhenAConnectionIsLostCloseChannel(OpenChannelContext): def when_connection_is_closed(self): - try: + with suppress(Exception): self.connection.protocol.connection_lost(Exception()) - except Exception: - pass self.tick() self.was_closed = self.channel.is_closed() diff --git a/test/connection_tests.py b/test/connection_tests.py index c0c76e2..03d52d5 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,5 +1,6 @@ import asyncio import sys +from contextlib import suppress import contexts from asynqp import spec, exceptions from asynqp.connection import open_connection @@ -107,10 +108,8 @@ def it_MUST_be_discarded(self): class WhenAConnectionIsLostCloseConnection(OpenConnectionContext): def when_connection_is_closed(self): - try: + with suppress(Exception): self.connection.protocol.connection_lost(Exception()) - except Exception: - pass def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) @@ -125,10 +124,8 @@ def given_a_channel(self): self.channel = self.wait_for(task) def when_server_closes_transport(self): - try: + with suppress(exceptions.ConnectionLostError): self.protocol.connection_lost(None) - except exceptions.ConnectionLostError: - pass def it_should_raise_error_in_connection_methods(self): try: diff --git a/test/queue_tests.py b/test/queue_tests.py index 0bac897..ddfcc02 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -1,5 +1,6 @@ import asyncio from datetime import datetime +from contextlib import suppress import contexts import asynqp from asynqp import message @@ -194,10 +195,8 @@ def given_I_asked_for_a_message(self): def when_connection_is_closed(self): # XXX: remove if we change behaviour to not raise - try: + with suppress(Exception): self.server.protocol.connection_lost(Exception()) - except Exception: - pass self.tick() def it_should_raise_exception(self): @@ -414,10 +413,8 @@ def given_a_consumer(self): self.consumer = task.result() def when_connection_is_closed(self): - try: + with suppress(Exception): self.connection.protocol.connection_lost(Exception()) - except Exception: - pass def it_should_not_hang(self): self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) From 2fe71095005a4a0b08e60aa40bd5281735c3d3b8 Mon Sep 17 00:00:00 2001 From: Kirill Pinchuk Date: Tue, 30 Aug 2016 18:15:57 +0300 Subject: [PATCH 110/118] Mark amq.rabbitmq.reply-to as valid queue name (#86) --- src/asynqp/channel.py | 1 - src/asynqp/queue.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 9669520..988c43d 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -13,7 +13,6 @@ from .log import log -VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) VALID_EXCHANGE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])+$', flags=re.A) diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index eac96ef..ca7b7da 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -283,7 +283,7 @@ def __init__(self, sender, synchroniser, reader, consumers, *, loop): @asyncio.coroutine def declare(self, name, durable, exclusive, auto_delete, passive, nowait, arguments): - if not VALID_QUEUE_NAME_RE.match(name): + if not (name == 'amq.rabbitmq.reply-to' or VALID_QUEUE_NAME_RE.match(name)): raise ValueError( "Not a valid queue name.\n" "Valid names consist of letters, digits, hyphen, underscore, " From 992c356b55292b3dc9f6e3bec10b889b7ba36266 Mon Sep 17 00:00:00 2001 From: Aksarin Mikhail Date: Thu, 20 Oct 2016 22:25:15 +0500 Subject: [PATCH 111/118] Add simple on_connection_close handler without exception handlers (#88) * ADD: connection close callback feature * FIX: forgoten return --- src/asynqp/__init__.py | 16 +++++++++++----- src/asynqp/protocol.py | 7 ++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index d457d88..572180a 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -22,7 +22,8 @@ def connect(host='localhost', port=5672, username='guest', password='guest', - virtual_host='/', *, + virtual_host='/', + on_connection_close=None, *, loop=None, sock=None, **kwargs): """ Connect to an AMQP server on the given host and port. @@ -35,6 +36,7 @@ def connect(host='localhost', :param str username: the username to authenticate with. :param str password: the password to authenticate with. :param str virtual_host: the AMQP virtual host to connect to. + :param func on_connection_close: function called after connection lost. :keyword BaseEventLoop loop: An instance of :class:`~asyncio.BaseEventLoop` to use. (Defaults to :func:`asyncio.get_event_loop()`) :keyword socket sock: A :func:`~socket.socket` instance to use for the connection. @@ -60,7 +62,10 @@ def connect(host='localhost', kwargs['sock'] = sock dispatcher = Dispatcher() - transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), **kwargs) + + def protocol_factory(): + return AMQP(dispatcher, loop, close_callback=on_connection_close) + transport, protocol = yield from loop.create_connection(protocol_factory, **kwargs) # RPC-like applications require TCP_NODELAY in order to acheive # minimal response time. Actually, this library send data in one @@ -85,7 +90,8 @@ def connect(host='localhost', def connect_and_open_channel(host='localhost', port=5672, username='guest', password='guest', - virtual_host='/', *, + virtual_host='/', + on_connection_close=None, *, loop=None, **kwargs): """ Connect to an AMQP server and open a channel on the connection. @@ -97,10 +103,10 @@ def connect_and_open_channel(host='localhost', Equivalent to:: - connection = yield from connect(host, port, username, password, virtual_host, loop=loop, **kwargs) + connection = yield from connect(host, port, username, password, virtual_host, on_connection_close, loop=loop, **kwargs) channel = yield from connection.open_channel() return connection, channel """ - connection = yield from connect(host, port, username, password, virtual_host, loop=loop, **kwargs) + connection = yield from connect(host, port, username, password, virtual_host, on_connection_close, loop=loop, **kwargs) channel = yield from connection.open_channel() return connection, channel diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index fa1157b..3104e0e 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -8,12 +8,13 @@ class AMQP(asyncio.Protocol): - def __init__(self, dispatcher, loop): + def __init__(self, dispatcher, loop, close_callback=None): self.dispatcher = dispatcher self.partial_frame = b'' self.frame_reader = FrameReader() self.heartbeat_monitor = HeartbeatMonitor(self, loop) self._closed = False + self._close_callback = close_callback def connection_made(self, transport): self.transport = transport @@ -51,6 +52,10 @@ def start_heartbeat(self, heartbeat_interval): def connection_lost(self, exc): # If self._closed=True - we closed the transport ourselves. No need to # dispatch PoisonPillFrame, as we should have closed everything already + if self._close_callback: + # _close_callback now only accepts coroutines + asyncio.async(self._close_callback(exc)) + if not self._closed: poison_exc = ConnectionLostError( 'The connection was unexpectedly lost', exc) From 6d05656af415d03cc8b86ae4fe67f89b02fd8f58 Mon Sep 17 00:00:00 2001 From: "R. David Murray" Date: Mon, 6 Feb 2017 11:30:30 -0500 Subject: [PATCH 112/118] Doc mandatory param; cross link to set_return_handler (#93) I spent a while digging around in the source before I found that one gets to handle unpublishable messages by setting a return handler on the channel. A link from publish to that method would have saved me a lot of time :) --- src/asynqp/exchange.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/asynqp/exchange.py b/src/asynqp/exchange.py index 531c7e7..894b78c 100644 --- a/src/asynqp/exchange.py +++ b/src/asynqp/exchange.py @@ -38,6 +38,7 @@ def publish(self, message, routing_key, *, mandatory=True): :param asynqp.Message message: the message to send :param str routing_key: the routing key with which to publish the message + :param bool mandatory: if True (the default) undeliverable messages result in an error (see also :meth:`Channel.set_return_handler`) """ self.sender.send_BasicPublish(self.name, routing_key, mandatory, message) From 53d97249ba1c230ba1e3c3f94dc7e0cd210d9832 Mon Sep 17 00:00:00 2001 From: Roman Tolkachyov Date: Fri, 8 Sep 2017 22:04:16 +0300 Subject: [PATCH 113/118] fix asyncio.async deprecation warning (replaced with ensure_future) (#98) * fix asyncio.async deprecation warning (replaced with ensure_future) * fix indentation --- src/asynqp/protocol.py | 6 ++--- test/base_contexts.py | 16 ++++++------- test/channel_tests.py | 10 ++++---- test/connection_tests.py | 8 +++---- test/exchange_tests.py | 14 +++++------ test/message_tests.py | 4 ++-- test/queue_tests.py | 50 ++++++++++++++++++++-------------------- test/util.py | 2 +- 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index 3104e0e..fbf8ee3 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -54,7 +54,7 @@ def connection_lost(self, exc): # dispatch PoisonPillFrame, as we should have closed everything already if self._close_callback: # _close_callback now only accepts coroutines - asyncio.async(self._close_callback(exc)) + asyncio.ensure_future(self._close_callback(exc)) if not self._closed: poison_exc = ConnectionLostError( @@ -121,8 +121,8 @@ def __init__(self, protocol, loop): def start(self, interval): if interval <= 0: return - self.send_hb_task = asyncio.async(self.send_heartbeat(interval), loop=self.loop) - self.monitor_task = asyncio.async(self.monitor_heartbeat(interval), loop=self.loop) + self.send_hb_task = asyncio.ensure_future(self.send_heartbeat(interval), loop=self.loop) + self.monitor_task = asyncio.ensure_future(self.monitor_heartbeat(interval), loop=self.loop) def stop(self): if self.send_hb_task is not None: diff --git a/test/base_contexts.py b/test/base_contexts.py index 682f048..adcaf2d 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -29,7 +29,7 @@ def async_partial(self, coro): Schedule a coroutine which you are not expecting to complete before the end of the test. Disables the error log when the task is destroyed before completing. """ - t = asyncio.async(coro) + t = asyncio.ensure_future(coro) t._log_destroy_pending = False self.tick() return t @@ -50,7 +50,7 @@ def given_a_mock_server_on_the_other_end_of_the_transport(self): class OpenConnectionContext(MockServerContext): def given_an_open_connection(self): connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} - task = asyncio.async(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) + task = asyncio.ensure_future(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.tick() start_method = spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US') @@ -75,7 +75,7 @@ def given_an_open_channel(self): self.channel = self.open_channel() def open_channel(self, channel_id=1): - task = asyncio.async(self.connection.open_channel(), loop=self.loop) + task = asyncio.ensure_future(self.connection.open_channel(), loop=self.loop) self.tick() self.server.send_method(channel_id, spec.ChannelOpenOK('')) return self.loop.run_until_complete(task) @@ -84,7 +84,7 @@ def open_channel(self, channel_id=1): class QueueContext(OpenChannelContext): def given_a_queue(self): queue_name = 'my.nice.queue' - task = asyncio.async(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), loop=self.loop) + task = asyncio.ensure_future(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.QueueDeclareOK(queue_name, 123, 456)) self.queue = task.result() @@ -95,8 +95,8 @@ def given_an_exchange(self): self.exchange = self.make_exchange('my.nice.exchange') def make_exchange(self, name): - task = asyncio.async(self.channel.declare_exchange(name, 'fanout', durable=True, auto_delete=False, internal=False), - loop=self.loop) + task = asyncio.ensure_future(self.channel.declare_exchange(name, 'fanout', durable=True, auto_delete=False, internal=False), + loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.ExchangeDeclareOK()) return task.result() @@ -104,7 +104,7 @@ def make_exchange(self, name): class BoundQueueContext(QueueContext, ExchangeContext): def given_a_bound_queue(self): - task = asyncio.async(self.queue.bind(self.exchange, 'routing.key')) + task = asyncio.ensure_future(self.queue.bind(self.exchange, 'routing.key')) self.tick() self.server.send_method(self.channel.id, spec.QueueBindOK()) self.binding = task.result() @@ -115,7 +115,7 @@ def given_a_consumer(self): self.callback = mock.Mock() del self.callback._is_coroutine # :( - task = asyncio.async(self.queue.consume(self.callback, no_local=False, no_ack=False, exclusive=False)) + task = asyncio.ensure_future(self.queue.consume(self.callback, no_local=False, no_ack=False, exclusive=False)) self.tick() self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) self.consumer = task.result() diff --git a/test/channel_tests.py b/test/channel_tests.py index 8e20532..9e716fa 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -30,7 +30,7 @@ def it_should_send_a_channel_open_frame_for_channel_2(self): class WhenChannelOpenOKArrives(OpenConnectionContext): def given_the_user_has_called_open_channel(self): - self.task = asyncio.async(self.connection.open_channel()) + self.task = asyncio.ensure_future(self.connection.open_channel()) self.tick() def when_channel_open_ok_arrives(self): @@ -109,7 +109,7 @@ def it_should_not_throw_an_exception(self): class WhenAnUnexpectedChannelCloseArrives(OpenChannelContext): def given_we_are_awaiting_QueueDeclareOK(self): - self.task = asyncio.async(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.task = asyncio.ensure_future(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) self.tick() def when_ChannelClose_arrives(self): @@ -133,7 +133,7 @@ def it_should_send_BasicQos_with_default_values(self): class WhenBasicQOSOkArrives(OpenChannelContext): def given_we_are_setting_qos_settings(self): - self.task = asyncio.async(self.channel.set_qos(prefetch_size=1000, prefetch_count=100, apply_globally=True)) + self.task = asyncio.ensure_future(self.channel.set_qos(prefetch_size=1000, prefetch_count=100, apply_globally=True)) self.tick() def when_BasicQosOk_arrives(self): @@ -250,7 +250,7 @@ def if_should_have_closed_channel(self): class WhenWeCloseConnectionChannelShouldAlsoClose(OpenChannelContext): def when_connection_is_closed(self): - self.task = asyncio.async(self.connection.close(), loop=self.loop) + self.task = asyncio.ensure_future(self.connection.close(), loop=self.loop) self.server.send_method(0, spec.ConnectionCloseOK()) self.tick() self.was_closed = self.channel.is_closed() @@ -280,7 +280,7 @@ def if_should_have_closed_channel(self): class WhenServerAndClientCloseChannelAtATime(OpenChannelContext): def when_both_sides_close_channel(self): # Client tries to close connection - self.task = asyncio.async(self.channel.close(), loop=self.loop) + self.task = asyncio.ensure_future(self.channel.close(), loop=self.loop) self.tick() # Before OK arrives server closes connection self.server.send_method( diff --git a/test/connection_tests.py b/test/connection_tests.py index 03d52d5..24a33b1 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -66,7 +66,7 @@ def it_should_send_ConnectionClose_with_no_exception(self): class WhenRecievingConnectionCloseOK(OpenConnectionContext): def given_a_connection_that_I_closed(self): - asyncio.async(self.connection.close()) + asyncio.ensure_future(self.connection.close()) self.tick() def when_connection_close_ok_arrives(self): @@ -79,7 +79,7 @@ def it_should_close_the_transport(self): class WhenAConnectionThatIsClosingReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): - t = asyncio.async(self.connection.close()) + t = asyncio.ensure_future(self.connection.close()) t._log_destroy_pending = False self.tick() self.server.reset() @@ -138,7 +138,7 @@ def it_should_raise_error_in_connection_methods(self): class WhenOpeningAChannelOnAClosedConnection(OpenConnectionContext): def when_client_closes_connection(self): - task = asyncio.async(self.connection.close()) + task = asyncio.ensure_future(self.connection.close()) self.tick() self.server.send_method(0, spec.ConnectionCloseOK()) self.tick() @@ -153,7 +153,7 @@ def it_should_raise_error_in_connection_methods(self): class WhenServerAndClientCloseConnectionAtATime(OpenConnectionContext): def when_both_sides_close_channel(self): # Client tries to close connection - self.task = asyncio.async(self.connection.close(), loop=self.loop) + self.task = asyncio.ensure_future(self.connection.close(), loop=self.loop) self.tick() # Before OK arrives server closes connection self.server.send_method( diff --git a/test/exchange_tests.py b/test/exchange_tests.py index ac302e3..64c40df 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -20,7 +20,7 @@ def it_should_send_ExchangeDeclare(self): class WhenExchangeDeclareOKArrives(OpenChannelContext): def given_I_declared_an_exchange(self): - self.task = asyncio.async(self.channel.declare_exchange('my.nice.exchange', 'fanout', durable=True, auto_delete=False, internal=False)) + self.task = asyncio.ensure_future(self.channel.declare_exchange('my.nice.exchange', 'fanout', durable=True, auto_delete=False, internal=False)) self.tick() def when_the_reply_arrives(self): @@ -49,7 +49,7 @@ def it_should_not_be_internal(self): class WhenIDeclareTheDefaultExchange(OpenChannelContext): def when_I_declare_an_exchange_with_an_empty_name(self): self.server.reset() - task = asyncio.async(self.channel.declare_exchange('', 'direct', durable=True, auto_delete=False, internal=False)) + task = asyncio.ensure_future(self.channel.declare_exchange('', 'direct', durable=True, auto_delete=False, internal=False)) self.tick() self.exchange = task.result() @@ -79,7 +79,7 @@ def examples_of_bad_words(cls): yield "contains'illegal$ymbols" def because_I_try_to_declare_the_exchange(self, name): - task = asyncio.async(self.channel.declare_exchange(name, 'direct')) + task = asyncio.ensure_future(self.channel.declare_exchange(name, 'direct')) self.tick() self.exception = task.exception() @@ -167,7 +167,7 @@ def it_should_send_ExchangeDelete(self): class WhenExchangeDeleteOKArrives(ExchangeContext): def given_I_deleted_the_exchange(self): - asyncio.async(self.exchange.delete(if_unused=True), loop=self.loop) + asyncio.ensure_future(self.exchange.delete(if_unused=True), loop=self.loop) self.tick() def when_confirmation_arrives(self): @@ -179,7 +179,7 @@ def it_should_not_throw(self): class WhenExchangeDeclareWithPassiveAndOKArrives(OpenChannelContext): def given_I_declared_an_exchange(self): - self.task = asyncio.async( + self.task = asyncio.ensure_future( self.channel.declare_exchange( 'name_1', 'fanout', passive=True, durable=True, auto_delete=False, internal=False)) @@ -201,7 +201,7 @@ def it_should_have_sent_passive_in_frame(self): class WhenExchangeDeclareWithPassiveAndErrorArrives(OpenChannelContext): def given_I_declared_an_exchange(self): - self.task = asyncio.async( + self.task = asyncio.ensure_future( self.channel.declare_exchange( 'name_1', 'fanout', passive=True, durable=True, auto_delete=False, internal=False)) @@ -217,7 +217,7 @@ def it_should_raise_not_found_error(self): class WhenIDeclareExchangeWithNoWait(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): - self.task = asyncio.async(self.channel.declare_exchange( + self.task = asyncio.ensure_future(self.channel.declare_exchange( 'my.nice.exchange', 'fanout', durable=True, auto_delete=False, internal=False, nowait=True), loop=self.loop) self.tick() diff --git a/test/message_tests.py b/test/message_tests.py index ab61a22..13fb050 100644 --- a/test/message_tests.py +++ b/test/message_tests.py @@ -128,7 +128,7 @@ def given_I_received_a_message(self): self.delivery_tag = 12487 msg = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - task = asyncio.async(self.queue.get()) + task = asyncio.ensure_future(self.queue.get()) self.tick() method = spec.BasicGetOK(self.delivery_tag, False, 'my.exchange', 'routing.key', 0) self.server.send_method(self.channel.id, method) @@ -154,7 +154,7 @@ def given_I_received_a_message(self): self.delivery_tag = 12487 msg = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - task = asyncio.async(self.queue.get()) + task = asyncio.ensure_future(self.queue.get()) self.tick() method = spec.BasicGetOK(self.delivery_tag, False, 'my.exchange', 'routing.key', 0) self.server.send_method(self.channel.id, method) diff --git a/test/queue_tests.py b/test/queue_tests.py index ddfcc02..91cbab4 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -23,7 +23,7 @@ def it_should_send_a_QueueDeclare_method(self): class WhenQueueDeclareOKArrives(OpenChannelContext): def given_I_declared_a_queue(self): self.queue_name = 'my.nice.queue' - self.task = asyncio.async(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) + self.task = asyncio.ensure_future(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) self.tick() def when_QueueDeclareOK_arrives(self): @@ -45,7 +45,7 @@ def it_should_auto_delete(self): class WhenILetTheServerPickTheQueueName(OpenChannelContext): def given_I_declared_a_queue(self): - self.task = asyncio.async(self.channel.declare_queue('', durable=True, exclusive=True, auto_delete=True), loop=self.loop) + self.task = asyncio.ensure_future(self.channel.declare_queue('', durable=True, exclusive=True, auto_delete=True), loop=self.loop) self.tick() self.queue_name = 'randomly.generated.name' @@ -65,8 +65,8 @@ def examples_of_bad_names(cls): yield 'contains~illegal/symbols' def when_I_declare_the_queue(self, queue_name): - self.task = asyncio.async(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), - loop=self.loop) + self.task = asyncio.ensure_future(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), + loop=self.loop) self.tick() def it_should_throw_ValueError(self): @@ -92,7 +92,7 @@ def it_should_throw_InvalidExchangeName(self): class WhenQueueBindOKArrives(QueueContext, ExchangeContext): def given_I_sent_QueueBind(self): - self.task = asyncio.async(self.queue.bind(self.exchange, 'routing.key')) + self.task = asyncio.ensure_future(self.queue.bind(self.exchange, 'routing.key')) self.tick() def when_QueueBindOK_arrives(self): @@ -117,7 +117,7 @@ def it_should_send_QueueUnbind(self): class WhenQueueUnbindOKArrives(BoundQueueContext): def given_I_unbound_the_queue(self): - self.task = asyncio.async(self.binding.unbind()) + self.task = asyncio.ensure_future(self.binding.unbind()) self.tick() def when_QueueUnbindOK_arrives(self): @@ -129,12 +129,12 @@ def it_should_be_ok(self): class WhenIUnbindAQueueTwice(BoundQueueContext): def given_an_unbound_queue(self): - asyncio.async(self.binding.unbind()) + asyncio.ensure_future(self.binding.unbind()) self.tick() self.server.send_method(self.channel.id, spec.QueueUnbindOK()) def when_I_unbind_the_queue_again(self): - self.task = asyncio.async(self.binding.unbind()) + self.task = asyncio.ensure_future(self.binding.unbind()) self.tick() def it_should_throw_Deleted(self): @@ -151,7 +151,7 @@ def it_should_send_BasicGet(self): class WhenBasicGetEmptyArrives(QueueContext): def given_I_asked_for_a_message(self): - self.task = asyncio.async(self.queue.get(no_ack=False)) + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) self.tick() def when_BasicGetEmpty_arrives(self): @@ -164,7 +164,7 @@ def it_should_return_None(self): class WhenBasicGetOKArrives(QueueContext): def given_I_asked_for_a_message(self): self.expected_message = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - self.task = asyncio.async(self.queue.get(no_ack=False)) + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) self.tick() def when_BasicGetOK_arrives_with_content(self): @@ -190,7 +190,7 @@ def it_should_put_the_routing_key_on_the_msg(self): class WhenConnectionClosedOnGet(QueueContext): def given_I_asked_for_a_message(self): - self.task = asyncio.async(self.queue.get(no_ack=False)) + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) self.tick() def when_connection_is_closed(self): @@ -213,7 +213,7 @@ def it_should_send_BasicConsume(self): class WhenConsumeOKArrives(QueueContext): def given_I_started_a_consumer(self): - self.task = asyncio.async(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + self.task = asyncio.ensure_future(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() def when_BasicConsumeOK_arrives(self): @@ -283,7 +283,7 @@ def it_should_send_a_BasicCancel_method(self): class WhenCancelOKArrives(ConsumerContext): def given_I_cancelled_a_consumer(self): - asyncio.async(self.consumer.cancel()) + asyncio.ensure_future(self.consumer.cancel()) self.tick() def when_BasicCancelOK_arrives(self): @@ -296,11 +296,11 @@ def it_should_be_cancelled(self): class WhenCancelOKArrivesForAConsumerWithAnOnCancelMethod(QueueContext): def given_I_started_and_cancelled_a_consumer(self): self.consumer = self.ConsumerWithOnCancel() - task = asyncio.async(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + task = asyncio.ensure_future(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) self.tick() - asyncio.async(task.result().cancel()) + asyncio.ensure_future(task.result().cancel()) self.tick() def when_BasicCancelOK_arrives(self): @@ -323,7 +323,7 @@ def on_cancel(self): class WhenAConsumerWithAnOnCancelMethodIsKilledDueToAnError(QueueContext): def given_I_started_a_consumer(self): self.consumer = self.ConsumerWithOnError() - asyncio.async(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + asyncio.ensure_future(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) self.tick() @@ -357,7 +357,7 @@ def it_should_send_a_QueuePurge_method(self): class WhenQueuePurgeOKArrives(QueueContext): def given_I_called_queue_purge(self): - self.task = asyncio.async(self.queue.purge()) + self.task = asyncio.ensure_future(self.queue.purge()) self.tick() def when_QueuePurgeOK_arrives(self): @@ -377,7 +377,7 @@ def it_should_send_a_QueueDelete_method(self): class WhenQueueDeleteOKArrives(QueueContext): def given_I_deleted_a_queue(self): - asyncio.async(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) + asyncio.ensure_future(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) self.tick() def when_QueueDeleteOK_arrives(self): @@ -390,12 +390,12 @@ def it_should_be_deleted(self): class WhenITryToUseADeletedQueue(QueueContext): def given_a_deleted_queue(self): - asyncio.async(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) + asyncio.ensure_future(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.QueueDeleteOK(123)) def when_I_try_to_use_the_queue(self): - self.task = asyncio.async(self.queue.get()) + self.task = asyncio.ensure_future(self.queue.get()) self.tick() def it_should_throw_Deleted(self): @@ -404,7 +404,7 @@ def it_should_throw_Deleted(self): class WhenAConnectionIsClosedCancelConsuming(QueueContext, ExchangeContext): def given_a_consumer(self): - task = asyncio.async(self.queue.consume( + task = asyncio.ensure_future(self.queue.consume( lambda x: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() @@ -422,7 +422,7 @@ def it_should_not_hang(self): class WhenIDeclareQueueWithPassiveAndOKArrives(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): - self.task = asyncio.async(self.channel.declare_queue( + self.task = asyncio.ensure_future(self.channel.declare_queue( '123', durable=True, exclusive=True, auto_delete=False, passive=True), loop=self.loop) self.tick() @@ -444,7 +444,7 @@ def it_should_have_sent_passive_in_frame(self): class WhenIDeclareQueueWithPassiveAndErrorArrives(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): - self.task = asyncio.async(self.channel.declare_queue( + self.task = asyncio.ensure_future(self.channel.declare_queue( '123', durable=True, exclusive=True, auto_delete=True, passive=True), loop=self.loop) self.tick() @@ -459,7 +459,7 @@ def it_should_raise_exception(self): class WhenIDeclareQueueWithNoWait(OpenChannelContext): def given_I_declared_a_queue_with_passive(self): - self.task = asyncio.async(self.channel.declare_queue( + self.task = asyncio.ensure_future(self.channel.declare_queue( '123', durable=True, exclusive=True, auto_delete=False, nowait=True), loop=self.loop) self.tick() @@ -477,7 +477,7 @@ def it_should_have_sent_nowait_in_frame(self): class WhenConsumerIsClosedServerSide(QueueContext, ExchangeContext): def given_a_consumer(self): - task = asyncio.async(self.queue.consume(lambda x: None)) + task = asyncio.ensure_future(self.queue.consume(lambda x: None)) self.tick() self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) self.tick() diff --git a/test/util.py b/test/util.py index 0821a49..f671b88 100644 --- a/test/util.py +++ b/test/util.py @@ -110,7 +110,7 @@ def __eq__(self, other): @contextmanager def silence_expected_destroy_pending_log(expected_coro_name=''): - real_async = asyncio.async + real_async = asyncio.ensure_future def async(*args, **kwargs): t = real_async(*args, **kwargs) From 62945369315c5ea185e22778670b9d5aa32bc7cd Mon Sep 17 00:00:00 2001 From: Evgeny Myzgin Date: Wed, 22 Aug 2018 01:54:09 +0300 Subject: [PATCH 114/118] Add support for Python up to 3.7. (#104) --- src/asynqp/channel.py | 8 ++++---- src/asynqp/connection.py | 8 ++++---- src/asynqp/exchange.py | 2 +- src/asynqp/queue.py | 16 ++++++++-------- src/asynqp/routing.py | 2 +- test/base_contexts.py | 5 ++--- test/integration_tests.py | 4 ++-- test/util.py | 19 +++++++++++++++++-- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 988c43d..09f6e33 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -85,7 +85,7 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, name, type, passive, durable, auto_delete, internal, nowait, arguments or {}) if not nowait: - yield from self.synchroniser.await(spec.ExchangeDeclareOK) + yield from self.synchroniser.wait(spec.ExchangeDeclareOK) self.reader.ready() ex = exchange.Exchange( self.reader, self.synchroniser, self.sender, name, type, durable, @@ -149,7 +149,7 @@ def set_qos(self, prefetch_size=0, prefetch_count=0, apply_globally=False): global=true to mean that the QoS settings should apply per-channel. """ self.sender.send_BasicQos(prefetch_size, prefetch_count, apply_globally) - yield from self.synchroniser.await(spec.BasicQosOK) + yield from self.synchroniser.wait(spec.BasicQosOK) self.reader.ready() def set_return_handler(self, handler): @@ -184,7 +184,7 @@ def close(self): self.sender.send_Close( 0, 'Channel closed by application', 0, 0) try: - yield from self.synchroniser.await(spec.ChannelCloseOK) + yield from self.synchroniser.wait(spec.ChannelCloseOK) except AMQPError: # For example if both sides want to close or the connection # is closed. @@ -232,7 +232,7 @@ def open(self): try: sender.send_ChannelOpen() reader.ready() - yield from synchroniser.await(spec.ChannelOpenOK) + yield from synchroniser.wait(spec.ChannelOpenOK) except: # don't rollback self.next_channel_id; # another call may have entered this method diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index c2e29a2..401e000 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -81,7 +81,7 @@ def close(self): self.sender.send_Close( 0, 'Connection closed by application', 0, 0) try: - yield from self.synchroniser.await(spec.ConnectionCloseOK) + yield from self.synchroniser.wait(spec.ConnectionCloseOK) except AMQPConnectionError: # For example if both sides want to close or the connection # is closed. @@ -107,7 +107,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): protocol.send_protocol_header() reader.ready() - yield from synchroniser.await(spec.ConnectionStart) + yield from synchroniser.wait(spec.ConnectionStart) sender.send_StartOK( {"product": "asynqp", "version": "0.1", # todo: use pkg_resources to inspect the package @@ -121,7 +121,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): ) reader.ready() - frame = yield from synchroniser.await(spec.ConnectionTune) + frame = yield from synchroniser.wait(spec.ConnectionTune) # just agree with whatever the server wants. Make this configurable in future connection_info['frame_max'] = frame.payload.frame_max heartbeat_interval = frame.payload.heartbeat @@ -131,7 +131,7 @@ def open_connection(loop, transport, protocol, dispatcher, connection_info): protocol.start_heartbeat(heartbeat_interval) reader.ready() - yield from synchroniser.await(spec.ConnectionOpenOK) + yield from synchroniser.wait(spec.ConnectionOpenOK) reader.ready() except: dispatcher.remove_handler(0) diff --git a/src/asynqp/exchange.py b/src/asynqp/exchange.py index 894b78c..bcb7b61 100644 --- a/src/asynqp/exchange.py +++ b/src/asynqp/exchange.py @@ -53,5 +53,5 @@ def delete(self, *, if_unused=True): it has no queues bound to it. """ self.sender.send_ExchangeDelete(self.name, if_unused) - yield from self.synchroniser.await(spec.ExchangeDeleteOK) + yield from self.synchroniser.wait(spec.ExchangeDeleteOK) self.reader.ready() diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index ca7b7da..bbd07cf 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -72,7 +72,7 @@ def bind(self, exchange, routing_key, *, arguments=None): raise InvalidExchangeName("Can't bind queue {} to the default exchange".format(self.name)) self.sender.send_QueueBind(self.name, exchange.name, routing_key, arguments or {}) - yield from self.synchroniser.await(spec.QueueBindOK) + yield from self.synchroniser.wait(spec.QueueBindOK) b = QueueBinding(self.reader, self.sender, self.synchroniser, self, exchange, routing_key) self.reader.ready() return b @@ -107,7 +107,7 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, ar raise Deleted("Queue {} was deleted".format(self.name)) self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive, arguments or {}) - tag = yield from self.synchroniser.await(spec.BasicConsumeOK) + tag = yield from self.synchroniser.wait(spec.BasicConsumeOK) consumer = Consumer( tag, callback, self.sender, self.synchroniser, self.reader, loop=self._loop) @@ -131,7 +131,7 @@ def get(self, *, no_ack=False): raise Deleted("Queue {} was deleted".format(self.name)) self.sender.send_BasicGet(self.name, no_ack) - tag_msg = yield from self.synchroniser.await(spec.BasicGetOK, spec.BasicGetEmpty) + tag_msg = yield from self.synchroniser.wait(spec.BasicGetOK, spec.BasicGetEmpty) if tag_msg is not None: consumer_tag, msg = tag_msg @@ -149,7 +149,7 @@ def purge(self): This method is a :ref:`coroutine `. """ self.sender.send_QueuePurge(self.name) - yield from self.synchroniser.await(spec.QueuePurgeOK) + yield from self.synchroniser.wait(spec.QueuePurgeOK) self.reader.ready() @asyncio.coroutine @@ -168,7 +168,7 @@ def delete(self, *, if_unused=True, if_empty=True): raise Deleted("Queue {} was already deleted".format(self.name)) self.sender.send_QueueDelete(self.name, if_unused, if_empty) - yield from self.synchroniser.await(spec.QueueDeleteOK) + yield from self.synchroniser.wait(spec.QueueDeleteOK) self.deleted = True self.reader.ready() @@ -216,7 +216,7 @@ def unbind(self, arguments=None): raise Deleted("Queue {} was already unbound from exchange {}".format(self.queue.name, self.exchange.name)) self.sender.send_QueueUnbind(self.queue.name, self.exchange.name, self.routing_key, arguments or {}) - yield from self.synchroniser.await(spec.QueueUnbindOK) + yield from self.synchroniser.wait(spec.QueueUnbindOK) self.deleted = True self.reader.ready() @@ -260,7 +260,7 @@ def cancel(self): """ self.sender.send_BasicCancel(self.tag) try: - yield from self.synchroniser.await(spec.BasicCancelOK) + yield from self.synchroniser.wait(spec.BasicCancelOK) except AMQPError: pass else: @@ -295,7 +295,7 @@ def declare(self, name, durable, exclusive, auto_delete, passive, nowait, self.sender.send_QueueDeclare( name, durable, exclusive, auto_delete, passive, nowait, arguments) if not nowait: - name = yield from self.synchroniser.await(spec.QueueDeclareOK) + name = yield from self.synchroniser.wait(spec.QueueDeclareOK) self.reader.ready() q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete, arguments, diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 637f97b..fb5cda2 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -56,7 +56,7 @@ def __init__(self, *, loop): self._futures = collections.defaultdict(collections.deque) self.connection_exc = None - def await(self, *expected_methods): + def wait(self, *expected_methods): fut = asyncio.Future(loop=self._loop) if self.connection_exc is not None: diff --git a/test/base_contexts.py b/test/base_contexts.py index adcaf2d..7603776 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -1,12 +1,11 @@ import asyncio import asynqp import asynqp.routing -from asyncio import test_utils from asynqp import spec from asynqp import protocol from asynqp.connection import open_connection from unittest import mock -from .util import MockServer, FakeTransport +from .util import MockServer, FakeTransport, run_briefly class LoopContext: @@ -22,7 +21,7 @@ def exception_handler(self, loop, context): self.exceptions.append(context['exception']) def tick(self): - test_utils.run_briefly(self.loop) + run_briefly(self.loop) def async_partial(self, coro): """ diff --git a/test/integration_tests.py b/test/integration_tests.py index e6bc297..1aa1f80 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -2,7 +2,7 @@ import asynqp import socket import contexts -from asyncio import test_utils +from .util import run_briefly class ConnectionContext: @@ -287,7 +287,7 @@ def when_I_consume_a_message(self): consumer = self.loop.run_until_complete( self.queue.consume(self.result.append, exclusive=True)) for retry in range(10): - test_utils.run_briefly(self.loop) + run_briefly(self.loop) if self.result: break consumer.cancel() diff --git a/test/util.py b/test/util.py index f671b88..aa6b1f9 100644 --- a/test/util.py +++ b/test/util.py @@ -112,11 +112,26 @@ def __eq__(self, other): def silence_expected_destroy_pending_log(expected_coro_name=''): real_async = asyncio.ensure_future - def async(*args, **kwargs): + def async_wrapper(*args, **kwargs): t = real_async(*args, **kwargs) if expected_coro_name in repr(t): t._log_destroy_pending = False return t - with mock.patch.object(asyncio, 'async', async): + with mock.patch.object(asyncio, 'ensure_future', async_wrapper): yield + + +def run_briefly(loop): + @asyncio.coroutine + def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() From 014a897128f4ed08b48c7da407f50964fe320e97 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 20 Jan 2019 16:44:18 -0500 Subject: [PATCH 115/118] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 86996fc..f6d8b3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: - python setup.py develop script: - - flake8 src test --ignore=E501,W503 + - flake8 src test --ignore=E501,W503,E722 - coverage run --source=src -m contexts -v - pushd doc && make html && popd From 858249ee07e7dfa892d7f2b70ab011f102c75159 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 20 Jan 2019 16:46:35 -0500 Subject: [PATCH 116/118] Update .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f6d8b3d..d340379 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ language: python python: - 3.4 - 3.5 + - 3.6 + - 3.7 services: rabbitmq From ce7ec392542daeaf7629cad13408a5966413cd5d Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 20 Jan 2019 16:49:00 -0500 Subject: [PATCH 117/118] Update .travis.yml --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d340379..0b9aea1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - 3.4 - 3.5 - 3.6 - - 3.7 services: rabbitmq From ea8630d1803d10d4fd64b1a0e50f3097710b34d1 Mon Sep 17 00:00:00 2001 From: Benjamin Hodgson Date: Sun, 20 Jan 2019 16:49:29 -0500 Subject: [PATCH 118/118] increment version to 0.6 --- doc/conf.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index c966a22..f0f017b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '0.5.1' +version = '0.6' # The full version, including alpha/beta/rc tags. -release = '0.5.1' +release = '0.6' def hide_class_constructor(app, what, name, obj, options, signature, return_annotation): diff --git a/setup.py b/setup.py index 3ff694c..4084722 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='asynqp', - version='0.5.1', + version='0.6', author="Benjamin Hodgson", author_email="benjamin.hodgson@huddle.net", url="https://github.com/benjamin-hodgson/asynqp",