diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/.gitignore b/.gitignore index 8875eb4..c0a52f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,56 +1,57 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -bin/ -build/ -develop-eggs/ -dist/ -eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.cache -nosetests.xml -coverage.xml - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# Rope -.ropeproject - -# Django stuff: -*.log -*.pot - -# Sphinx documentation -doc/_build/ - -*.sublime-workspace -.ipynb_checkpoints +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +doc/_build/ + +*.sublime-workspace +.ipynb_checkpoints +.DS_Store diff --git a/.travis.yml b/.travis.yml index 8200de4..0b9aea1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,25 @@ +sudo: false + language: python -python: 3.4 +python: + - 3.4 + - 3.5 + - 3.6 services: rabbitmq install: - - pip install contexts colorama sphinx - - python setup.py install + - pip install -r requirements.txt + - python setup.py develop script: - - run-contexts -v - - cd doc && make html + - flake8 src test --ignore=E501,W503,E722 + - coverage run --source=src -m contexts -v + - pushd doc && make html && popd + +after_success: + coveralls deploy: provider: pypi @@ -21,6 +30,3 @@ deploy: tags: true all_branches: true distributions: "sdist bdist_wheel bdist_egg" - -after_deploy: - - curl -X POST http://readthedocs.org/build/asynqp diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dceb3b4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,35 @@ +What's new in `asynqp` +====================== + + +v0.5 +---- +* Channels will no longer break if their calls are cancelled. Issue #52. +* Fixed message properties decoding without `content_type`. Pull request #66. +* Added `nowait` argument to Exchange and Queue declarations. +* Added `passive` argument to Exchange and Queue declarations. +* Added `on_error` and `on_cancel` callbacks for Consumer. Issue #34 +* Changed the closing scheme for Channel/Connection. Proper exceptions are now +always propagated to user. Issues #57, #58 +* Complete internals refactor and cleanup. Rull requests #48, #49, #50. +* Add NO_DELAY option for socket. Issue #40. (Thanks to @socketpair for PR #41) +* Change heartbeat to be a proper background task. Issue #45. +* `loop` is now proparly passed to all components from open_connection call. Pull request #42. +* Add support for Python up to 3.5. + +v0.4 +---- + +* Improved error handling. + * When the connection to the server is lost, any futures awaiting communication + from the server will now be cancelled. + (Thanks to @lenzenmi, in pull request #19) + * More detailed exceptions on channel closure. +* Support for custom RabbitMQ extensions by an `arguments` keyword parameter for a number of methods. + (Thanks to @fweisel, in pull request #27) +* Improved compatibility with RabbitMQ's implementation of the + wire protocol, including better support for tables. + (Thanks to @fweisel, in pull requests #24, #25, #26, #28) +* Support for a `sock` keyword argument to `asynqp.connect`. + (Thanks to @urbaniak, in pull request #14) + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 182eae9..baa3bf4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,6 +5,10 @@ To contribute to this project, submit a [pull request](https://help.github.com/a 3. [Open a pull request](https://help.github.com/articles/creating-a-pull-request) in this repo to merge your topic branch into the mainstream 4. I'll review your changes and merge your pull request as soon as possible +If you want to contribute to the project, but are not sure what you want to work on, +I am always happy for help on any of the [open issues](https://github.com/benjamin-hodgson/asynqp/issues) +in the GitHub tracker. + This project is built using Test-Driven-Development. So if you're planning to contribute a feature or bugfix, please **ensure that it is covered by tests** before submitting it for review. Use your best judgment to diff --git a/LICENSE b/LICENSE index 3db08be..00694a5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2014 Benjamin Hodgson +Copyright (c) 2013-2015 Benjamin Hodgson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5e0aa3b..3e5f1b9 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ asynqp ====== [![Build Status](https://travis-ci.org/benjamin-hodgson/asynqp.svg?branch=master)](https://travis-ci.org/benjamin-hodgson/asynqp) -[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.3)](https://readthedocs.org/projects/asynqp/?badge=v0.3) +[![Documentation Status](https://readthedocs.org/projects/asynqp/badge/?version=v0.4)](https://readthedocs.org/projects/asynqp/?badge=v0.4) +[![Coverage Status](https://coveralls.io/repos/benjamin-hodgson/asynqp/badge.svg?branch=master&service=github)](https://coveralls.io/github/benjamin-hodgson/asynqp?branch=master) +[![Requirements Status](https://requires.io/github/benjamin-hodgson/asynqp/requirements.svg?branch=master)](https://requires.io/github/benjamin-hodgson/asynqp/requirements/?branch=master) `asynqp` is an AMQP (aka [RabbitMQ](rabbitmq.com)) client library for Python 3.4's new [`asyncio`](https://docs.python.org/3.4/library/asyncio.html) module. @@ -19,7 +21,10 @@ import asynqp @asyncio.coroutine -def send_and_receive(): +def hello_world(): + """ + Sends a 'hello world' message and then reads it from the queue. + """ # connect to the RabbitMQ broker connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') @@ -34,7 +39,7 @@ def send_and_receive(): yield from queue.bind(exchange, 'routing.key') # If you pass in a dict it will be automatically converted to JSON - msg = asynqp.Message({'test_body': 'content'}) + msg = asynqp.Message({'hello': 'world'}) exchange.publish(msg, 'routing.key') # Synchronously get a message from the queue @@ -50,7 +55,7 @@ def send_and_receive(): if __name__ == "__main__": loop = asyncio.get_event_loop() - loop.run_until_complete(send_and_receive()) + loop.run_until_complete(hello_world()) ``` diff --git a/RELEASING b/RELEASING index d15d97d..6163b2b 100644 --- a/RELEASING +++ b/RELEASING @@ -1,8 +1,8 @@ -1. Check Travis is green on the latest commit -2. Increment the version in setup.py -3. Increment the version in doc/conf.py -4. Commit and tag -5. Push using --follow-tags -6. Check that the tests passed on Travis and it published to the Cheese Shop -6a. If it failed to publish to the cheese shop for some reason, run `python setup.py register` followed by `python setup.py sdist bdist_egg bdist_wheel upload` -7. Build the docs at readthedocs.org and increment the latest version +1. Check Travis is green on the latest commit +2. Increment the version in setup.py +3. Increment the version in doc/conf.py +4. Commit and tag +5. Push using --follow-tags +6. Check that the tests passed on Travis and it published to the Cheese Shop +6a. If it failed to publish to the cheese shop for some reason, run `python setup.py register` followed by `python setup.py sdist bdist_egg bdist_wheel upload` +7. Build the docs at readthedocs.org and increment the latest version diff --git a/TODO b/TODO index 3fdf635..f9b5f50 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ API design issues Public implementation-related fields in public classes (even though they're not documented) - Threading issues + Threading issues? Load testing? @@ -41,6 +41,5 @@ Unimplemented methods: Unimplemented functions Customise connection-tune response - General exception handling Passive declares No-wait flags diff --git a/doc/conf.py b/doc/conf.py index 396667a..f0f017b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,16 +51,16 @@ # General information about the project. project = 'asynqp' -copyright = '2014, Benjamin Hodgson' +copyright = '2014-2015, Benjamin Hodgson' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.3' +version = '0.6' # The full version, including alpha/beta/rc tags. -release = '0.3' +release = '0.6' def hide_class_constructor(app, what, name, obj, options, signature, return_annotation): diff --git a/doc/conformance.rst b/doc/conformance.rst index 3ebfd9e..70fe83b 100644 --- a/doc/conformance.rst +++ b/doc/conformance.rst @@ -46,7 +46,7 @@ that are currently supported by ``asynqp``. +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | exchange | | :orange:`partial` | :class:`asynqp.Exchange` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ -| | declare/declare-ok | :orange:`partial` | :meth:`asynqp.Channel.declare_exchange` | Not all parameters presently supported | +| | declare/declare-ok | :green:`full` | :meth:`asynqp.Channel.declare_exchange` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | | delete/delete-ok | :green:`full` | :meth:`asynqp.Exchange.delete` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ @@ -56,7 +56,7 @@ that are currently supported by ``asynqp``. +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | queue | | :orange:`partial` | :class:`asynqp.Queue` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ -| | declare/declare-ok | :orange:`partial` | :meth:`asynqp.Channel.declare_queue` | Not all parameters presently supported | +| | declare/declare-ok | :green:`full` | :meth:`asynqp.Channel.declare_queue` | | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ | | bind/bind-ok | :orange:`partial` | :meth:`asynqp.Queue.bind` | Not all parameters presently supported | +------------+----------------------+-------------------+-------------------------------------------+-----------------------------------------+ diff --git a/doc/examples.rst b/doc/examples.rst new file mode 100644 index 0000000..d5d8d2e --- /dev/null +++ b/doc/examples.rst @@ -0,0 +1,14 @@ +Examples +======== + + +Hello World +----------- +.. literalinclude:: /examples/helloworld.py + :language: python + + +Reconnecting +------------ +.. literalinclude:: /examples/reconnecting.py + :language: python diff --git a/doc/examples/helloworld.py b/doc/examples/helloworld.py new file mode 100644 index 0000000..896e9b7 --- /dev/null +++ b/doc/examples/helloworld.py @@ -0,0 +1,40 @@ +import asyncio +import asynqp + + +@asyncio.coroutine +def hello_world(): + """ + Sends a 'hello world' message and then reads it from the queue. + """ + # connect to the RabbitMQ broker + connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') + + # Open a communications channel + channel = yield from connection.open_channel() + + # Create a queue and an exchange on the broker + exchange = yield from channel.declare_exchange('test.exchange', 'direct') + queue = yield from channel.declare_queue('test.queue') + + # Bind the queue to the exchange, so the queue will get messages published to the exchange + yield from queue.bind(exchange, 'routing.key') + + # If you pass in a dict it will be automatically converted to JSON + msg = asynqp.Message({'hello': 'world'}) + exchange.publish(msg, 'routing.key') + + # Synchronously get a message from the queue + received_message = yield from queue.get() + print(received_message.json()) # get JSON from incoming messages easily + + # Acknowledge a delivered message + received_message.ack() + + yield from channel.close() + yield from connection.close() + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(hello_world()) diff --git a/doc/examples/reconnecting.py b/doc/examples/reconnecting.py new file mode 100644 index 0000000..577506d --- /dev/null +++ b/doc/examples/reconnecting.py @@ -0,0 +1,92 @@ +import asyncio +import asynqp +import logging + +logging.basicConfig(level=logging.INFO) + +RECONNECT_BACKOFF = 1 + + +class Consumer: + + def __init__(self, connection, queue): + self.queue = queue + self.connection = connection + + def __call__(self, msg): + self.queue.put_nowait(msg) + + def on_error(self, exc): + print("Connection lost while consuming queue", exc) + + +@asyncio.coroutine +def connect_and_consume(queue): + # connect to the RabbitMQ broker + connection = yield from asynqp.connect( + 'localhost', 5672, username='guest', password='guest') + try: + channel = yield from connection.open_channel() + amqp_queue = yield from channel.declare_queue('test.queue') + consumer = Consumer(connection, queue) + yield from amqp_queue.consume(consumer) + except asynqp.AMQPError as err: + print("Could not consume on queue", err) + yield from connection.close() + return None + return connection + + +@asyncio.coroutine +def reconnector(queue): + try: + connection = None + while True: + if connection is None or connection.is_closed(): + print("Connecting to rabbitmq...") + try: + connection = yield from connect_and_consume(queue) + except (ConnectionError, OSError): + print("Failed to connect to rabbitmq server. " + "Will retry in {} seconds".format(RECONNECT_BACKOFF)) + connection = None + if connection is None: + yield from asyncio.sleep(RECONNECT_BACKOFF) + else: + print("Successfully connected and consuming test.queue") + # poll connection state every 100ms + yield from asyncio.sleep(0.1) + except asyncio.CancelledError: + if connection is not None: + yield from connection.close() + + +@asyncio.coroutine +def process_msgs(queue): + try: + while True: + msg = yield from queue.get() + print("Received", msg.body) + msg.ack() + except asyncio.CancelledError: + pass + + +def main(): + loop = asyncio.get_event_loop() + queue = asyncio.Queue() + # Start main indexing task in the background + reconnect_task = loop.create_task(reconnector(queue)) + process_task = loop.create_task(process_msgs(queue)) + try: + loop.run_forever() + except KeyboardInterrupt: + process_task.cancel() + reconnect_task.cancel() + loop.run_until_complete(process_task) + loop.run_until_complete(reconnect_task) + loop.close() + + +if __name__ == "__main__": + main() diff --git a/doc/extensions.rst b/doc/extensions.rst new file mode 100644 index 0000000..55b93fd --- /dev/null +++ b/doc/extensions.rst @@ -0,0 +1,11 @@ +.. _extensions: + +Protocol extensions +=================== + +RabbitMQ, and other brokers, support certain extensions to the AMQP protocol. +`asynqp`'s support for such extensions currently includes +*optional extra arguments* to certain methods such as :meth:`Channel.declare_queue() `. + +The acceptable parameters for optional argument dictionaries is implementation-dependent. +See`RabbitMQ's supported extensions `. diff --git a/doc/index.rst b/doc/index.rst index 3882b05..e3e402c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,8 +1,3 @@ -.. asynqp documentation master file, created by - sphinx-quickstart on Wed Jun 11 20:44:10 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - ``asynqp`` ========== An AMQP (aka `RabbitMQ `_) client library for :mod:`asyncio`. @@ -10,44 +5,8 @@ An AMQP (aka `RabbitMQ `_) client library for :mod:`as Example ------- -:: - - import asyncio - import asynqp - - - @asyncio.coroutine - def send_and_receive(): - # connect to the RabbitMQ broker - connection = yield from asynqp.connect('localhost', 5672, username='guest', password='guest') - - # Open a communications channel - channel = yield from connection.open_channel() - - # Create a queue and an exchange on the broker - exchange = yield from channel.declare_exchange('test.exchange', 'direct') - queue = yield from channel.declare_queue('test.queue') - - # Bind the queue to the exchange, so the queue will get messages published to the exchange - yield from queue.bind(exchange, 'routing.key') - - # If you pass in a dict it will be automatically converted to JSON - msg = asynqp.Message({'test_body': 'content'}) - exchange.publish(msg, 'routing.key') - - # Synchronously get a message from the queue - received_message = yield from queue.get() - print(received_message.json()) # get JSON from incoming messages easily - - # Acknowledge a delivered message - received_message.ack() - - yield from connection.close() - - - if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(send_and_receive()) +.. literalinclude:: /examples/helloworld.py + :language: python Installation @@ -66,7 +25,9 @@ Table of contents :maxdepth: 2 reference + examples conformance + extensions * :ref:`genindex` * :ref:`modindex` diff --git a/doc/make.bat b/doc/make.bat index bc72244..388cf2c 100644 --- a/doc/make.bat +++ b/doc/make.bat @@ -1,242 +1,242 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\asynqp.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\asynqp.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\asynqp.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\asynqp.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/doc/reference.rst b/doc/reference.rst index 57583bb..2acef7e 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -66,3 +66,11 @@ Message objects .. autoclass:: IncomingMessage :members: + + +Exceptions +---------- + +.. automodule:: asynqp.exceptions + :members: + :undoc-members: diff --git a/ez_setup.py b/ez_setup.py index 1b8fd95..30d5fd0 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,18 +1,11 @@ #!/usr/bin/env python -"""Bootstrap setuptools installation -To use setuptools in your package's setup.py, include this -file in the same directory and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -To require a specific version of setuptools, set a download -mirror, or use an alternate download directory, simply supply -the appropriate options to ``use_setuptools()``. +""" +Setuptools bootstrapping installer. -This file can also be run as a script to install or upgrade setuptools. +Run this script to install or upgrade setuptools. """ + import os import shutil import sys @@ -23,19 +16,29 @@ import platform import textwrap import contextlib +import warnings from distutils import log +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + try: from site import USER_SITE except ImportError: USER_SITE = None -DEFAULT_VERSION = "3.4.4" +DEFAULT_VERSION = "15.2" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" +DEFAULT_SAVE_DIR = os.curdir + def _python_cmd(*args): """ + Execute a command. + Return True if the command succeeded. """ args = (sys.executable,) + args @@ -43,6 +46,7 @@ def _python_cmd(*args): def _install(archive_filename, install_args=()): + """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') @@ -54,6 +58,7 @@ def _install(archive_filename, install_args=()): def _build_egg(egg, archive_filename, to_dir): + """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) @@ -64,28 +69,36 @@ def _build_egg(egg, archive_filename, to_dir): raise IOError('Could not build the egg.') -def get_zip_class(): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ - class ContextualZipFile(zipfile.ZipFile): - def __enter__(self): - return self - def __exit__(self, type, value, traceback): - self.close - return zipfile.ZipFile if hasattr(zipfile.ZipFile, '__exit__') else \ - ContextualZipFile +class ContextualZipFile(zipfile.ZipFile): + + """Supplement ZipFile class to support context manager for Python 2.6.""" + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __new__(cls, *args, **kwargs): + """Construct a ZipFile or ContextualZipFile as appropriate.""" + if hasattr(zipfile.ZipFile, '__exit__'): + return zipfile.ZipFile(*args, **kwargs) + return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): - # extracting the archive + """ + Unzip filename to a temporary directory, set to the cwd. + + The unzipped target is cleaned up after. + """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) - with get_zip_class()(filename) as archive: + with ContextualZipFile(filename) as archive: archive.extractall() # going in the directory @@ -100,6 +113,7 @@ def archive_context(filename): def _do_download(version, download_base, to_dir, download_delay): + """Download Setuptools.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): @@ -117,41 +131,77 @@ def _do_download(version, download_base, to_dir, download_delay): setuptools.bootstrap_install_from = egg -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, download_delay=15): + """ + Ensure that a setuptools version is installed. + + Return None. Raise SystemExit if the requested version + or later cannot be installed. + """ to_dir = os.path.abspath(to_dir) + + # prior to importing, capture the module state for + # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) + try: import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: pkg_resources.require("setuptools>=" + version) + # a suitable version is already installed return + except ImportError: + # pkg_resources not available; setuptools is not installed; download + pass except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, download_delay) + # no version of setuptools was found; allow download + pass except pkg_resources.VersionConflict as VC_err: if imported: - msg = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. + _conflict_bail(VC_err, version) + + # otherwise, unload pkg_resources to allow the downloaded version to + # take precedence. + del pkg_resources + _unload_pkg_resources() + + return _do_download(version, download_base, to_dir, download_delay) - (Currently using {VC_err.args[0]!r}) - """).format(VC_err=VC_err, version=version) - sys.stderr.write(msg) - sys.exit(2) - # otherwise, reload ok - del pkg_resources, sys.modules['pkg_resources'] - return _do_download(version, download_base, to_dir, download_delay) +def _conflict_bail(VC_err, version): + """ + Setuptools was imported prior to invocation, so it is + unsafe to unload it. Bail out. + """ + conflict_tmpl = textwrap.dedent(""" + The required version of setuptools (>={version}) is not available, + and can't be installed while this script is running. Please + install a more recent version first, using + 'easy_install -U setuptools'. + + (Currently using {VC_err.args[0]!r}) + """) + msg = conflict_tmpl.format(**locals()) + sys.stderr.write(msg) + sys.exit(2) + + +def _unload_pkg_resources(): + del_modules = [ + name for name in sys.modules + if name.startswith('pkg_resources') + ] + for mod_name in del_modules: + del sys.modules[mod_name] + def _clean_check(cmd, target): """ - Run the command to download target. If the command fails, clean up before - re-raising the error. + Run the command to download target. + + If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) @@ -160,115 +210,110 @@ def _clean_check(cmd, target): os.unlink(target) raise + def download_file_powershell(url, target): """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. + Download the file at url to target using Powershell. + + Powershell will validate trust. + Raise an exception if the command cannot complete. """ target = os.path.abspath(target) + ps_cmd = ( + "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " + "[System.Net.CredentialCache]::DefaultCredentials; " + "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" + % vars() + ) cmd = [ 'powershell', '-Command', - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), + ps_cmd, ] _clean_check(cmd, target) + def has_powershell(): + """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_powershell.viable = has_powershell + def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) + def has_curl(): cmd = ['curl', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_curl.viable = has_curl + def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) + def has_wget(): cmd = ['wget', '--version'] - devnull = open(os.path.devnull, 'wb') - try: + with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False - finally: - devnull.close() return True - download_file_wget.viable = has_wget + def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - src = dst = None + """Use Python to download the file, without connection authentication.""" + src = urlopen(url) try: - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. + # Read all the data in one block. data = src.read() - dst = open(target, "wb") - dst.write(data) finally: - if src: - src.close() - if dst: - dst.close() + src.close() + # Write all the data in one block to avoid creating a partial file. + with open(target, "wb") as dst: + dst.write(data) download_file_insecure.viable = lambda: True + def get_best_downloader(): - downloaders = [ + downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, - ] + ) + viable_downloaders = (dl for dl in downloaders if dl.viable()) + return next(viable_downloaders, None) - for dl in downloaders: - if dl.viable(): - return dl -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=DEFAULT_SAVE_DIR, delay=15, + downloader_factory=get_best_downloader): """ - Download setuptools from a specified location and return its filename + Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end + as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. @@ -287,16 +332,18 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, downloader(url, saveto) return os.path.realpath(saveto) + def _build_install_args(options): """ - Build the arguments to 'python setup.py install' on the setuptools package + Build the arguments to 'python setup.py install' on the setuptools package. + + Returns list of command line arguments. """ return ['--user'] if options.user_install else [] + def _parse_args(): - """ - Parse the command line for options - """ + """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, @@ -314,18 +361,30 @@ def _parse_args(): '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) + parser.add_option( + '--to-dir', + help="Directory to save (and re-use) package", + default=DEFAULT_SAVE_DIR, + ) options, args = parser.parse_args() # positional arguments are ignored return options -def main(): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - archive = download_setuptools( + +def _download_args(options): + """Return args for download_setuptools function from cmdline args.""" + return dict( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, + to_dir=options.to_dir, ) + + +def main(): + """Install or upgrade setuptools and EasyInstall.""" + options = _parse_args() + archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..335d90c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +contexts +colorama +sphinx +flake8 +coverage +coveralls diff --git a/setup.py b/setup.py index 44b586f..4084722 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,15 @@ -from ez_setup import use_setuptools -use_setuptools() +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() - -from setuptools import setup, find_packages + from setuptools import setup, find_packages setup( name='asynqp', - version='0.3', + version='0.6', author="Benjamin Hodgson", author_email="benjamin.hodgson@huddle.net", url="https://github.com/benjamin-hodgson/asynqp", @@ -20,6 +22,7 @@ "Development Status :: 3 - Alpha", "Programming Language :: Python", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", "License :: OSI Approved :: MIT License", "Intended Audience :: Developers", "Intended Audience :: Information Technology", diff --git a/src/asynqp/__init__.py b/src/asynqp/__init__.py index d649527..572180a 100644 --- a/src/asynqp/__init__.py +++ b/src/asynqp/__init__.py @@ -1,5 +1,8 @@ +# flake8: noqa + +import socket import asyncio -from .exceptions import AMQPError, Deleted +from .exceptions import * # noqa from .message import Message, IncomingMessage from .connection import Connection from .channel import Channel @@ -7,12 +10,21 @@ from .queue import Queue, QueueBinding, Consumer +__all__ = [ + "Message", "IncomingMessage", + "Connection", "Channel", "Exchange", "Queue", "QueueBinding", "Consumer", + "connect", "connect_and_open_channel" +] +__all__ += exceptions.__all__ + + @asyncio.coroutine def connect(host='localhost', port=5672, username='guest', password='guest', - virtual_host='/', *, - loop=None, **kwargs): + virtual_host='/', + on_connection_close=None, *, + loop=None, sock=None, **kwargs): """ Connect to an AMQP server on the given host and port. @@ -24,21 +36,53 @@ def connect(host='localhost', :param str username: the username to authenticate with. :param str password: the password to authenticate with. :param str virtual_host: the AMQP virtual host to connect to. + :param func on_connection_close: function called after connection lost. + :keyword BaseEventLoop loop: An instance of :class:`~asyncio.BaseEventLoop` to use. + (Defaults to :func:`asyncio.get_event_loop()`) + :keyword socket sock: A :func:`~socket.socket` instance to use for the connection. + This is passed on to :meth:`loop.create_connection() `. + If ``sock`` is supplied then ``host`` and ``port`` will be ignored. + + Further keyword arguments are passed on to :meth:`loop.create_connection() `. - Further keyword arguments are passed on to :meth:`create_connection() `. + This function will set TCP_NODELAY on TCP and TCP6 sockets either on supplied ``sock`` or created one. :return: the :class:`Connection` object. """ from .protocol import AMQP from .routing import Dispatcher - from .connection import ConnectionInfo, open_connection + from .connection import open_connection loop = asyncio.get_event_loop() if loop is None else loop + if sock is None: + kwargs['host'] = host + kwargs['port'] = port + else: + kwargs['sock'] = sock + dispatcher = Dispatcher() - transport, protocol = yield from loop.create_connection(lambda: AMQP(dispatcher, loop), host=host, port=port, **kwargs) - connection = yield from open_connection(loop, protocol, dispatcher, ConnectionInfo(username, password, virtual_host)) + def protocol_factory(): + return AMQP(dispatcher, loop, close_callback=on_connection_close) + transport, protocol = yield from loop.create_connection(protocol_factory, **kwargs) + + # RPC-like applications require TCP_NODELAY in order to acheive + # minimal response time. Actually, this library send data in one + # big chunk and so this will not affect TCP-performance. + sk = transport.get_extra_info('socket') + # 1. Unfortunatelly we cannot check socket type (sk.type == socket.SOCK_STREAM). https://bugs.python.org/issue21327 + # 2. Proto remains zero, if not specified at creation of socket + if (sk.family in (socket.AF_INET, socket.AF_INET6)) and (sk.proto in (0, socket.IPPROTO_TCP)): + sk.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + connection_info = { + 'username': username, + 'password': password, + 'virtual_host': virtual_host + } + connection = yield from open_connection( + loop, transport, protocol, dispatcher, connection_info) return connection @@ -46,7 +90,8 @@ def connect(host='localhost', def connect_and_open_channel(host='localhost', port=5672, username='guest', password='guest', - virtual_host='/', *, + virtual_host='/', + on_connection_close=None, *, loop=None, **kwargs): """ Connect to an AMQP server and open a channel on the connection. @@ -58,10 +103,10 @@ def connect_and_open_channel(host='localhost', Equivalent to:: - connection = yield from connect(host, port, username, password, virtual_host, loop=loop, **kwargs) + connection = yield from connect(host, port, username, password, virtual_host, on_connection_close, loop=loop, **kwargs) channel = yield from connection.open_channel() return connection, channel """ - connection = yield from connect(host, port, username, password, virtual_host, loop=loop, **kwargs) + connection = yield from connect(host, port, username, password, virtual_host, on_connection_close, loop=loop, **kwargs) channel = yield from connection.open_channel() return connection, channel diff --git a/src/asynqp/_exceptions.py b/src/asynqp/_exceptions.py new file mode 100644 index 0000000..fee3757 --- /dev/null +++ b/src/asynqp/_exceptions.py @@ -0,0 +1,6 @@ +class AMQPError(IOError): + pass + + +class AMQPChannelError(AMQPError): + pass diff --git a/src/asynqp/amqp0-9-1.xml b/src/asynqp/amqp0-9-1.xml index 5ddab06..da785eb 100644 --- a/src/asynqp/amqp0-9-1.xml +++ b/src/asynqp/amqp0-9-1.xml @@ -1,2843 +1,2843 @@ - - - - - - - - - - - - - - - - - - - - - - - Indicates that the method completed successfully. This reply code is - reserved for future use - the current protocol design does not use positive - confirmation and reply codes are sent only in case of an error. - - - - - - The client attempted to transfer content larger than the server could accept - at the present time. The client may retry at a later time. - - - - - - When the exchange cannot deliver to a consumer when the immediate flag is - set. As a result of pending data on the queue or the absence of any - consumers of the queue. - - - - - - An operator intervened to close the connection for some reason. The client - may retry at some later date. - - - - - - The client tried to work with an unknown virtual host. - - - - - - The client attempted to work with a server entity to which it has no - access due to security settings. - - - - - - The client attempted to work with a server entity that does not exist. - - - - - - The client attempted to work with a server entity to which it has no - access because another client is working with it. - - - - - - The client requested a method that was not allowed because some precondition - failed. - - - - - - The sender sent a malformed frame that the recipient could not decode. - This strongly implies a programming error in the sending peer. - - - - - - The sender sent a frame that contained illegal values for one or more - fields. This strongly implies a programming error in the sending peer. - - - - - - The client sent an invalid sequence of frames, attempting to perform an - operation that was considered invalid by the server. This usually implies - a programming error in the client. - - - - - - The client attempted to work with a channel that had not been correctly - opened. This most likely indicates a fault in the client layer. - - - - - - The peer sent a frame that was not expected, usually in the context of - a content header and body. This strongly indicates a fault in the peer's - content processing. - - - - - - The server could not complete the method because it lacked sufficient - resources. This may be due to the client creating too many of some type - of entity. - - - - - - The client tried to work with some entity in a manner that is prohibited - by the server, due to security settings or by some other criteria. - - - - - - The client tried to use functionality that is not implemented in the - server. - - - - - - The server could not complete the method because of an internal error. - The server may require intervention by an operator in order to resume - normal operations. - - - - - - - - - - Identifier for the consumer, valid within the current channel. - - - - - - The server-assigned and channel-specific delivery tag - - - - The delivery tag is valid only within the channel from which the message was - received. I.e. a client MUST NOT receive a message on one channel and then - acknowledge it on another. - - - - - The server MUST NOT use a zero value for delivery tags. Zero is reserved - for client use, meaning "all messages so far received". - - - - - - - The exchange name is a client-selected string that identifies the exchange for - publish methods. - - - - - - - - - - If this field is set the server does not expect acknowledgements for - messages. That is, when a message is delivered to the client the server - assumes the delivery will succeed and immediately dequeues it. This - functionality may increase performance but at the cost of reliability. - Messages can get lost if a client dies before they are delivered to the - application. - - - - - - If the no-local field is set the server will not send messages to the connection that - published them. - - - - - - If set, the server will not respond to the method. The client should not wait - for a reply method. If the server could not complete the method it will raise a - channel or connection exception. - - - - - - Unconstrained. - - - - - - - - This table provides a set of peer properties, used for identification, debugging, - and general information. - - - - - - The queue name identifies the queue within the vhost. In methods where the queue - name may be blank, and that has no specific significance, this refers to the - 'current' queue for the channel, meaning the last queue that the client declared - on the channel. If the client did not declare a queue, and the method needs a - queue name, this will result in a 502 (syntax error) channel exception. - - - - - - - - This indicates that the message has been previously delivered to this or - another client. - - - - The server SHOULD try to signal redelivered messages when it can. When - redelivering a message that was not successfully acknowledged, the server - SHOULD deliver it to the original client if possible. - - - Declare a shared queue and publish a message to the queue. Consume the - message using explicit acknowledgements, but do not acknowledge the - message. Close the connection, reconnect, and consume from the queue - again. The message should arrive with the redelivered flag set. - - - - - The client MUST NOT rely on the redelivered field but should take it as a - hint that the message may already have been processed. A fully robust - client must be able to track duplicate received messages on non-transacted, - and locally-transacted channels. - - - - - - - The number of messages in the queue, which will be zero for newly-declared - queues. This is the number of messages present in the queue, and committed - if the channel on which they were published is transacted, that are not - waiting acknowledgement. - - - - - - The reply code. The AMQ reply codes are defined as constants at the start - of this formal specification. - - - - - - - The localised reply text. This text can be logged as an aid to resolving - issues. - - - - - - - - - - - - - - - - - - - - The connection class provides methods for a client to establish a network connection to - a server, and for both peers to operate the connection thereafter. - - - - connection = open-connection *use-connection close-connection - open-connection = C:protocol-header - S:START C:START-OK - *challenge - S:TUNE C:TUNE-OK - C:OPEN S:OPEN-OK - challenge = S:SECURE C:SECURE-OK - use-connection = *channel - close-connection = C:CLOSE S:CLOSE-OK - / S:CLOSE C:CLOSE-OK - - - - - - - - - - This method starts the connection negotiation process by telling the client the - protocol version that the server proposes, along with a list of security mechanisms - which the client can use for authentication. - - - - - If the server cannot support the protocol specified in the protocol header, - it MUST respond with a valid protocol header and then close the socket - connection. - - - The client sends a protocol header containing an invalid protocol name. - The server MUST respond by sending a valid protocol header and then closing - the connection. - - - - - The server MUST provide a protocol version that is lower than or equal to - that requested by the client in the protocol header. - - - The client requests a protocol version that is higher than any valid - implementation, e.g. 2.0. The server must respond with a protocol header - indicating its supported protocol version, e.g. 1.0. - - - - - If the client cannot handle the protocol version suggested by the server - it MUST close the socket connection without sending any further data. - - - The server sends a protocol version that is lower than any valid - implementation, e.g. 0.1. The client must respond by closing the - connection without sending any further data. - - - - - - - - - The major version number can take any value from 0 to 99 as defined in the - AMQP specification. - - - - - - The minor version number can take any value from 0 to 99 as defined in the - AMQP specification. - - - - - - - The properties SHOULD contain at least these fields: "host", specifying the - server host name or address, "product", giving the name of the server product, - "version", giving the name of the server version, "platform", giving the name - of the operating system, "copyright", if appropriate, and "information", giving - other general information. - - - Client connects to server and inspects the server properties. It checks for - the presence of the required fields. - - - - - - - A list of the security mechanisms that the server supports, delimited by spaces. - - - - - - - A list of the message locales that the server supports, delimited by spaces. The - locale defines the language in which the server will send reply texts. - - - - The server MUST support at least the en_US locale. - - - Client connects to server and inspects the locales field. It checks for - the presence of the required locale(s). - - - - - - - - - This method selects a SASL security mechanism. - - - - - - - - - The properties SHOULD contain at least these fields: "product", giving the name - of the client product, "version", giving the name of the client version, "platform", - giving the name of the operating system, "copyright", if appropriate, and - "information", giving other general information. - - - - - - - A single security mechanisms selected by the client, which must be one of those - specified by the server. - - - - The client SHOULD authenticate using the highest-level security profile it - can handle from the list provided by the server. - - - - - If the mechanism field does not contain one of the security mechanisms - proposed by the server in the Start method, the server MUST close the - connection without sending any further data. - - - Client connects to server and sends an invalid security mechanism. The - server must respond by closing the connection (a socket close, with no - connection close negotiation). - - - - - - - - A block of opaque data passed to the security mechanism. The contents of this - data are defined by the SASL security mechanism. - - - - - - - A single message locale selected by the client, which must be one of those - specified by the server. - - - - - - - - - - The SASL protocol works by exchanging challenges and responses until both peers have - received sufficient information to authenticate each other. This method challenges - the client to provide more information. - - - - - - - - Challenge information, a block of opaque binary data passed to the security - mechanism. - - - - - - - This method attempts to authenticate, passing a block of SASL data for the security - mechanism at the server side. - - - - - - - A block of opaque data passed to the security mechanism. The contents of this - data are defined by the SASL security mechanism. - - - - - - - - - - This method proposes a set of connection configuration values to the client. The - client can accept and/or adjust these. - - - - - - - - - Specifies highest channel number that the server permits. Usable channel numbers - are in the range 1..channel-max. Zero indicates no specified limit. - - - - - - The largest frame size that the server proposes for the connection, including - frame header and end-byte. The client can negotiate a lower value. Zero means - that the server does not impose any specific limit but may reject very large - frames if it cannot allocate resources for them. - - - - Until the frame-max has been negotiated, both peers MUST accept frames of up - to frame-min-size octets large, and the minimum negotiated value for frame-max - is also frame-min-size. - - - Client connects to server and sends a large properties field, creating a frame - of frame-min-size octets. The server must accept this frame. - - - - - - - The delay, in seconds, of the connection heartbeat that the server wants. - Zero means the server does not want a heartbeat. - - - - - - - This method sends the client's connection tuning parameters to the server. - Certain fields are negotiated, others provide capability information. - - - - - - - The maximum total number of channels that the client will use per connection. - - - - If the client specifies a channel max that is higher than the value provided - by the server, the server MUST close the connection without attempting a - negotiated close. The server may report the error in some fashion to assist - implementors. - - - - - - - - - The largest frame size that the client and server will use for the connection. - Zero means that the client does not impose any specific limit but may reject - very large frames if it cannot allocate resources for them. Note that the - frame-max limit applies principally to content frames, where large contents can - be broken into frames of arbitrary size. - - - - Until the frame-max has been negotiated, both peers MUST accept frames of up - to frame-min-size octets large, and the minimum negotiated value for frame-max - is also frame-min-size. - - - - - If the client specifies a frame max that is higher than the value provided - by the server, the server MUST close the connection without attempting a - negotiated close. The server may report the error in some fashion to assist - implementors. - - - - - - - The delay, in seconds, of the connection heartbeat that the client wants. Zero - means the client does not want a heartbeat. - - - - - - - - - This method opens a connection to a virtual host, which is a collection of - resources, and acts to separate multiple application domains within a server. - The server may apply arbitrary limits per virtual host, such as the number - of each type of entity that may be used, per connection and/or in total. - - - - - - - - The name of the virtual host to work with. - - - - If the server supports multiple virtual hosts, it MUST enforce a full - separation of exchanges, queues, and all associated entities per virtual - host. An application, connected to a specific virtual host, MUST NOT be able - to access resources of another virtual host. - - - - - The server SHOULD verify that the client has permission to access the - specified virtual host. - - - - - - - - - - - - This method signals to the client that the connection is ready for use. - - - - - - - - - - - This method indicates that the sender wants to close the connection. This may be - due to internal conditions (e.g. a forced shut-down) or due to an error handling - a specific method, i.e. an exception. When a close is due to an exception, the - sender provides the class and method id of the method which caused the exception. - - - - After sending this method, any received methods except Close and Close-OK MUST - be discarded. The response to receiving a Close after sending Close must be to - send Close-Ok. - - - - - - - - - - - - - When the close is provoked by a method exception, this is the class of the - method. - - - - - - When the close is provoked by a method exception, this is the ID of the method. - - - - - - - This method confirms a Connection.Close method and tells the recipient that it is - safe to release resources for the connection and close the socket. - - - - A peer that detects a socket closure without having received a Close-Ok - handshake method SHOULD log the error. - - - - - - - - - - - - The channel class provides methods for a client to establish a channel to a - server and for both peers to operate the channel thereafter. - - - - channel = open-channel *use-channel close-channel - open-channel = C:OPEN S:OPEN-OK - use-channel = C:FLOW S:FLOW-OK - / S:FLOW C:FLOW-OK - / functional-class - close-channel = C:CLOSE S:CLOSE-OK - / S:CLOSE C:CLOSE-OK - - - - - - - - - - This method opens a channel to the server. - - - - The client MUST NOT use this method on an already-opened channel. - - - Client opens a channel and then reopens the same channel. - - - - - - - - - - - This method signals to the client that the channel is ready for use. - - - - - - - - - - - This method asks the peer to pause or restart the flow of content data sent by - a consumer. This is a simple flow-control mechanism that a peer can use to avoid - overflowing its queues or otherwise finding itself receiving more messages than - it can process. Note that this method is not intended for window control. It does - not affect contents returned by Basic.Get-Ok methods. - - - - - When a new channel is opened, it is active (flow is active). Some applications - assume that channels are inactive until started. To emulate this behaviour a - client MAY open the channel, then pause it. - - - - - - When sending content frames, a peer SHOULD monitor the channel for incoming - methods and respond to a Channel.Flow as rapidly as possible. - - - - - - A peer MAY use the Channel.Flow method to throttle incoming content data for - internal reasons, for example, when exchanging data over a slower connection. - - - - - - The peer that requests a Channel.Flow method MAY disconnect and/or ban a peer - that does not respect the request. This is to prevent badly-behaved clients - from overwhelming a server. - - - - - - - - - - - If 1, the peer starts sending content frames. If 0, the peer stops sending - content frames. - - - - - - - Confirms to the peer that a flow command was received and processed. - - - - - - Confirms the setting of the processed flow method: 1 means the peer will start - sending or continue to send content frames; 0 means it will not. - - - - - - - - - This method indicates that the sender wants to close the channel. This may be due to - internal conditions (e.g. a forced shut-down) or due to an error handling a specific - method, i.e. an exception. When a close is due to an exception, the sender provides - the class and method id of the method which caused the exception. - - - - After sending this method, any received methods except Close and Close-OK MUST - be discarded. The response to receiving a Close after sending Close must be to - send Close-Ok. - - - - - - - - - - - - - When the close is provoked by a method exception, this is the class of the - method. - - - - - - When the close is provoked by a method exception, this is the ID of the method. - - - - - - - This method confirms a Channel.Close method and tells the recipient that it is safe - to release resources for the channel. - - - - A peer that detects a socket closure without having received a Channel.Close-Ok - handshake method SHOULD log the error. - - - - - - - - - - - - Exchanges match and distribute messages across queues. Exchanges can be configured in - the server or declared at runtime. - - - - exchange = C:DECLARE S:DECLARE-OK - / C:DELETE S:DELETE-OK - - - - - - - - The server MUST implement these standard exchange types: fanout, direct. - - - Client attempts to declare an exchange with each of these standard types. - - - - - The server SHOULD implement these standard exchange types: topic, headers. - - - Client attempts to declare an exchange with each of these standard types. - - - - - The server MUST, in each virtual host, pre-declare an exchange instance - for each standard exchange type that it implements, where the name of the - exchange instance, if defined, is "amq." followed by the exchange type name. - - - The server MUST, in each virtual host, pre-declare at least two direct - exchange instances: one named "amq.direct", the other with no public name - that serves as a default exchange for Publish methods. - - - Client declares a temporary queue and attempts to bind to each required - exchange instance ("amq.fanout", "amq.direct", "amq.topic", and "amq.headers" - if those types are defined). - - - - - The server MUST pre-declare a direct exchange with no public name to act as - the default exchange for content Publish methods and for default queue bindings. - - - Client checks that the default exchange is active by specifying a queue - binding with no exchange name, and publishing a message with a suitable - routing key but without specifying the exchange name, then ensuring that - the message arrives in the queue correctly. - - - - - The server MUST NOT allow clients to access the default exchange except - by specifying an empty exchange name in the Queue.Bind and content Publish - methods. - - - - - The server MAY implement other exchange types as wanted. - - - - - - - - This method creates an exchange if it does not already exist, and if the exchange - exists, verifies that it is of the correct and expected class. - - - - The server SHOULD support a minimum of 16 exchanges per virtual host and - ideally, impose no limit except as defined by available resources. - - - The client declares as many exchanges as it can until the server reports - an error; the number of exchanges successfully declared must be at least - sixteen. - - - - - - - - - - - - - Exchange names starting with "amq." are reserved for pre-declared and - standardised exchanges. The client MAY declare an exchange starting with - "amq." if the passive option is set, or the exchange already exists. - - - The client attempts to declare a non-existing exchange starting with - "amq." and with the passive option set to zero. - - - - - The exchange name consists of a non-empty sequence of these characters: - letters, digits, hyphen, underscore, period, or colon. - - - The client attempts to declare an exchange with an illegal name. - - - - - - - - Each exchange belongs to one of a set of exchange types implemented by the - server. The exchange types define the functionality of the exchange - i.e. how - messages are routed through it. It is not valid or meaningful to attempt to - change the type of an existing exchange. - - - - Exchanges cannot be redeclared with different types. The client MUST not - attempt to redeclare an existing exchange with a different type than used - in the original Exchange.Declare method. - - - TODO. - - - - - The client MUST NOT attempt to declare an exchange with a type that the - server does not support. - - - TODO. - - - - - - - If set, the server will reply with Declare-Ok if the exchange already - exists with the same name, and raise an error if not. The client can - use this to check whether an exchange exists without modifying the - server state. When set, all other method fields except name and no-wait - are ignored. A declare with both passive and no-wait has no effect. - Arguments are compared for semantic equivalence. - - - - If set, and the exchange does not already exist, the server MUST - raise a channel exception with reply code 404 (not found). - - - TODO. - - - - - If not set and the exchange exists, the server MUST check that the - existing exchange has the same values for type, durable, and arguments - fields. The server MUST respond with Declare-Ok if the requested - exchange matches these fields, and MUST raise a channel exception if - not. - - - TODO. - - - - - - - If set when creating a new exchange, the exchange will be marked as durable. - Durable exchanges remain active when a server restarts. Non-durable exchanges - (transient exchanges) are purged if/when a server restarts. - - - - The server MUST support both durable and transient exchanges. - - - TODO. - - - - - - - - - - - - - A set of arguments for the declaration. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - This method confirms a Declare method and confirms the name of the exchange, - essential for automatically-named exchanges. - - - - - - - - - This method deletes an exchange. When an exchange is deleted all queue bindings on - the exchange are cancelled. - - - - - - - - - - - - The client MUST NOT attempt to delete an exchange that does not exist. - - - - - - - - If set, the server will only delete the exchange if it has no queue bindings. If - the exchange has queue bindings the server does not delete it but raises a - channel exception instead. - - - - The server MUST NOT delete an exchange that has bindings on it, if the if-unused - field is true. - - - The client declares an exchange, binds a queue to it, then tries to delete it - setting if-unused to true. - - - - - - - - - This method confirms the deletion of an exchange. - - - - - - - - - Queues store and forward messages. Queues can be configured in the server or created at - runtime. Queues must be attached to at least one exchange in order to receive messages - from publishers. - - - - queue = C:DECLARE S:DECLARE-OK - / C:BIND S:BIND-OK - / C:UNBIND S:UNBIND-OK - / C:PURGE S:PURGE-OK - / C:DELETE S:DELETE-OK - - - - - - - - - - This method creates or checks a queue. When creating a new queue the client can - specify various properties that control the durability of the queue and its - contents, and the level of sharing for the queue. - - - - - The server MUST create a default binding for a newly-declared queue to the - default exchange, which is an exchange of type 'direct' and use the queue - name as the routing key. - - - Client declares a new queue, and then without explicitly binding it to an - exchange, attempts to send a message through the default exchange binding, - i.e. publish a message to the empty exchange, with the queue name as routing - key. - - - - - - The server SHOULD support a minimum of 256 queues per virtual host and ideally, - impose no limit except as defined by available resources. - - - Client attempts to declare as many queues as it can until the server reports - an error. The resulting count must at least be 256. - - - - - - - - - - - - - The queue name MAY be empty, in which case the server MUST create a new - queue with a unique generated name and return this to the client in the - Declare-Ok method. - - - Client attempts to declare several queues with an empty name. The client then - verifies that the server-assigned names are unique and different. - - - - - Queue names starting with "amq." are reserved for pre-declared and - standardised queues. The client MAY declare a queue starting with - "amq." if the passive option is set, or the queue already exists. - - - The client attempts to declare a non-existing queue starting with - "amq." and with the passive option set to zero. - - - - - The queue name can be empty, or a sequence of these characters: - letters, digits, hyphen, underscore, period, or colon. - - - The client attempts to declare a queue with an illegal name. - - - - - - - If set, the server will reply with Declare-Ok if the queue already - exists with the same name, and raise an error if not. The client can - use this to check whether a queue exists without modifying the - server state. When set, all other method fields except name and no-wait - are ignored. A declare with both passive and no-wait has no effect. - Arguments are compared for semantic equivalence. - - - - The client MAY ask the server to assert that a queue exists without - creating the queue if not. If the queue does not exist, the server - treats this as a failure. - - - Client declares an existing queue with the passive option and expects - the server to respond with a declare-ok. Client then attempts to declare - a non-existent queue with the passive option, and the server must close - the channel with the correct reply-code. - - - - - If not set and the queue exists, the server MUST check that the - existing queue has the same values for durable, exclusive, auto-delete, - and arguments fields. The server MUST respond with Declare-Ok if the - requested queue matches these fields, and MUST raise a channel exception - if not. - - - TODO. - - - - - - - If set when creating a new queue, the queue will be marked as durable. Durable - queues remain active when a server restarts. Non-durable queues (transient - queues) are purged if/when a server restarts. Note that durable queues do not - necessarily hold persistent messages, although it does not make sense to send - persistent messages to a transient queue. - - - - The server MUST recreate the durable queue after a restart. - - - Client declares a durable queue. The server is then restarted. The client - then attempts to send a message to the queue. The message should be successfully - delivered. - - - - - The server MUST support both durable and transient queues. - - A client declares two named queues, one durable and one transient. - - - - - - - Exclusive queues may only be accessed by the current connection, and are - deleted when that connection closes. Passive declaration of an exclusive - queue by other connections are not allowed. - - - - - The server MUST support both exclusive (private) and non-exclusive (shared) - queues. - - - A client declares two named queues, one exclusive and one non-exclusive. - - - - - - The client MAY NOT attempt to use a queue that was declared as exclusive - by another still-open connection. - - - One client declares an exclusive queue. A second client on a different - connection attempts to declare, bind, consume, purge, delete, or declare - a queue of the same name. - - - - - - - If set, the queue is deleted when all consumers have finished using it. The last - consumer can be cancelled either explicitly or because its channel is closed. If - there was no consumer ever on the queue, it won't be deleted. Applications can - explicitly delete auto-delete queues using the Delete method as normal. - - - - - The server MUST ignore the auto-delete field if the queue already exists. - - - Client declares two named queues, one as auto-delete and one explicit-delete. - Client then attempts to declare the two queues using the same names again, - but reversing the value of the auto-delete field in each case. Verify that the - queues still exist with the original auto-delete flag values. - - - - - - - - - A set of arguments for the declaration. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - This method confirms a Declare method and confirms the name of the queue, essential - for automatically-named queues. - - - - - - - Reports the name of the queue. If the server generated a queue name, this field - contains that name. - - - - - - - - - Reports the number of active consumers for the queue. Note that consumers can - suspend activity (Channel.Flow) in which case they do not appear in this count. - - - - - - - - - This method binds a queue to an exchange. Until a queue is bound it will not - receive any messages. In a classic messaging model, store-and-forward queues - are bound to a direct exchange and subscription queues are bound to a topic - exchange. - - - - - A server MUST allow ignore duplicate bindings - that is, two or more bind - methods for a specific queue, with identical arguments - without treating these - as an error. - - - A client binds a named queue to an exchange. The client then repeats the bind - (with identical arguments). - - - - - - A server MUST not deliver the same message more than once to a queue, even if - the queue has multiple bindings that match the message. - - - A client declares a named queue and binds it using multiple bindings to the - amq.topic exchange. The client then publishes a message that matches all its - bindings. - - - - - - The server MUST allow a durable queue to bind to a transient exchange. - - - A client declares a transient exchange. The client then declares a named durable - queue and then attempts to bind the transient exchange to the durable queue. - - - - - - Bindings of durable queues to durable exchanges are automatically durable - and the server MUST restore such bindings after a server restart. - - - A server declares a named durable queue and binds it to a durable exchange. The - server is restarted. The client then attempts to use the queue/exchange combination. - - - - - - The server SHOULD support at least 4 bindings per queue, and ideally, impose no - limit except as defined by available resources. - - - A client declares a named queue and attempts to bind it to 4 different - exchanges. - - - - - - - - - - - - Specifies the name of the queue to bind. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to bind an unnamed queue. - - - - - The client MUST NOT attempt to bind a queue that does not exist. - - - The client attempts to bind a non-existent queue. - - - - - - - - A client MUST NOT be allowed to bind a queue to a non-existent exchange. - - - A client attempts to bind an named queue to a undeclared exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - - - Specifies the routing key for the binding. The routing key is used for routing - messages depending on the exchange configuration. Not all exchanges use a - routing key - refer to the specific exchange documentation. If the queue name - is empty, the server uses the last queue declared on the channel. If the - routing key is also empty, the server uses this queue name for the routing - key as well. If the queue name is provided but the routing key is empty, the - server does the binding with that empty routing key. The meaning of empty - routing keys depends on the exchange implementation. - - - - If a message queue binds to a direct exchange using routing key K and a - publisher sends the exchange a message with routing key R, then the message - MUST be passed to the message queue if K = R. - - - - - - - - - A set of arguments for the binding. The syntax and semantics of these arguments - depends on the exchange class. - - - - - - This method confirms that the bind was successful. - - - - - - - - This method unbinds a queue from an exchange. - - If a unbind fails, the server MUST raise a connection exception. - - - - - - - - - Specifies the name of the queue to unbind. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to unbind an unnamed queue. - - - - - The client MUST NOT attempt to unbind a queue that does not exist. - - - The client attempts to unbind a non-existent queue. - - - - - - The name of the exchange to unbind from. - - - The client MUST NOT attempt to unbind a queue from an exchange that - does not exist. - - - The client attempts to unbind a queue from a non-existent exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - - Specifies the routing key of the binding to unbind. - - - - Specifies the arguments of the binding to unbind. - - - - - This method confirms that the unbind was successful. - - - - - - - - This method removes all messages from a queue which are not awaiting - acknowledgment. - - - - - The server MUST NOT purge messages that have already been sent to a client - but not yet acknowledged. - - - - - - The server MAY implement a purge queue or log that allows system administrators - to recover accidentally-purged messages. The server SHOULD NOT keep purged - messages in the same storage spaces as the live messages since the volumes of - purged messages may get very large. - - - - - - - - - - - - Specifies the name of the queue to purge. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to purge an unnamed queue. - - - - - The client MUST NOT attempt to purge a queue that does not exist. - - - The client attempts to purge a non-existent queue. - - - - - - - - - This method confirms the purge of a queue. - - - - - - Reports the number of messages purged. - - - - - - - - - This method deletes a queue. When a queue is deleted any pending messages are sent - to a dead-letter queue if this is defined in the server configuration, and all - consumers on the queue are cancelled. - - - - - The server SHOULD use a dead-letter queue to hold messages that were pending on - a deleted queue, and MAY provide facilities for a system administrator to move - these messages back to an active queue. - - - - - - - - - - - - Specifies the name of the queue to delete. - - - The client MUST either specify a queue name or have previously declared a - queue on the same channel - - - The client opens a channel and attempts to delete an unnamed queue. - - - - - The client MUST NOT attempt to delete a queue that does not exist. - - - The client attempts to delete a non-existent queue. - - - - - - - If set, the server will only delete the queue if it has no consumers. If the - queue has consumers the server does does not delete it but raises a channel - exception instead. - - - - The server MUST NOT delete a queue that has consumers on it, if the if-unused - field is true. - - - The client declares a queue, and consumes from it, then tries to delete it - setting if-unused to true. - - - - - - - If set, the server will only delete the queue if it has no messages. - - - - The server MUST NOT delete a queue that has messages on it, if the - if-empty field is true. - - - The client declares a queue, binds it and publishes some messages into it, - then tries to delete it setting if-empty to true. - - - - - - - - - This method confirms the deletion of a queue. - - - - - Reports the number of messages deleted. - - - - - - - - - The Basic class provides methods that support an industry-standard messaging model. - - - - basic = C:QOS S:QOS-OK - / C:CONSUME S:CONSUME-OK - / C:CANCEL S:CANCEL-OK - / C:PUBLISH content - / S:RETURN content - / S:DELIVER content - / C:GET ( S:GET-OK content / S:GET-EMPTY ) - / C:ACK - / C:REJECT - / C:RECOVER-ASYNC - / C:RECOVER S:RECOVER-OK - - - - - - - - The server SHOULD respect the persistent property of basic messages and - SHOULD make a best-effort to hold persistent basic messages on a reliable - storage mechanism. - - - Send a persistent message to queue, stop server, restart server and then - verify whether message is still present. Assumes that queues are durable. - Persistence without durable queues makes no sense. - - - - - - The server MUST NOT discard a persistent basic message in case of a queue - overflow. - - - Declare a queue overflow situation with persistent messages and verify that - messages do not get lost (presumably the server will write them to disk). - - - - - - The server MAY use the Channel.Flow method to slow or stop a basic message - publisher when necessary. - - - Declare a queue overflow situation with non-persistent messages and verify - whether the server responds with Channel.Flow or not. Repeat with persistent - messages. - - - - - - The server MAY overflow non-persistent basic messages to persistent - storage. - - - - - - - The server MAY discard or dead-letter non-persistent basic messages on a - priority basis if the queue size exceeds some configured limit. - - - - - - - The server MUST implement at least 2 priority levels for basic messages, - where priorities 0-4 and 5-9 are treated as two distinct levels. - - - Send a number of priority 0 messages to a queue. Send one priority 9 - message. Consume messages from the queue and verify that the first message - received was priority 9. - - - - - - The server MAY implement up to 10 priority levels. - - - Send a number of messages with mixed priorities to a queue, so that all - priority values from 0 to 9 are exercised. A good scenario would be ten - messages in low-to-high priority. Consume from queue and verify how many - priority levels emerge. - - - - - - The server MUST deliver messages of the same priority in order irrespective of - their individual persistence. - - - Send a set of messages with the same priority but different persistence - settings to a queue. Consume and verify that messages arrive in same order - as originally published. - - - - - - The server MUST support un-acknowledged delivery of Basic content, i.e. - consumers with the no-ack field set to TRUE. - - - - - - The server MUST support explicitly acknowledged delivery of Basic content, - i.e. consumers with the no-ack field set to FALSE. - - - Declare a queue and a consumer using explicit acknowledgements. Publish a - set of messages to the queue. Consume the messages but acknowledge only - half of them. Disconnect and reconnect, and consume from the queue. - Verify that the remaining messages are received. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This method requests a specific quality of service. The QoS can be specified for the - current channel or for all channels on the connection. The particular properties and - semantics of a qos method always depend on the content class semantics. Though the - qos method could in principle apply to both peers, it is currently meaningful only - for the server. - - - - - - - - The client can request that messages be sent in advance so that when the client - finishes processing a message, the following message is already held locally, - rather than needing to be sent down the channel. Prefetching gives a performance - improvement. This field specifies the prefetch window size in octets. The server - will send a message in advance if it is equal to or smaller in size than the - available prefetch size (and also falls into other prefetch limits). May be set - to zero, meaning "no specific limit", although other prefetch limits may still - apply. The prefetch-size is ignored if the no-ack option is set. - - - - The server MUST ignore this setting when the client is not processing any - messages - i.e. the prefetch size does not limit the transfer of single - messages to a client, only the sending in advance of more messages while - the client still has one or more unacknowledged messages. - - - Define a QoS prefetch-size limit and send a single message that exceeds - that limit. Verify that the message arrives correctly. - - - - - - - Specifies a prefetch window in terms of whole messages. This field may be used - in combination with the prefetch-size field; a message will only be sent in - advance if both prefetch windows (and those at the channel and connection level) - allow it. The prefetch-count is ignored if the no-ack option is set. - - - - The server may send less data in advance than allowed by the client's - specified prefetch windows but it MUST NOT send more. - - - Define a QoS prefetch-size limit and a prefetch-count limit greater than - one. Send multiple messages that exceed the prefetch size. Verify that - no more than one message arrives at once. - - - - - - - By default the QoS settings apply to the current channel only. If this field is - set, they are applied to the entire connection. - - - - - - - This method tells the client that the requested QoS levels could be handled by the - server. The requested QoS applies to all active consumers until a new QoS is - defined. - - - - - - - - - This method asks the server to start a "consumer", which is a transient request for - messages from a specific queue. Consumers last as long as the channel they were - declared on, or until the client cancels them. - - - - - The server SHOULD support at least 16 consumers per queue, and ideally, impose - no limit except as defined by available resources. - - - Declare a queue and create consumers on that queue until the server closes the - connection. Verify that the number of consumers created was at least sixteen - and report the total number. - - - - - - - - - - - Specifies the name of the queue to consume from. - - - - - Specifies the identifier for the consumer. The consumer tag is local to a - channel, so two clients can use the same consumer tags. If this field is - empty the server will generate a unique tag. - - - - The client MUST NOT specify a tag that refers to an existing consumer. - - - Attempt to create two consumers with the same non-empty tag, on the - same channel. - - - - - The consumer tag is valid only within the channel from which the - consumer was created. I.e. a client MUST NOT create a consumer in one - channel and then use it in another. - - - Attempt to create a consumer in one channel, then use in another channel, - in which consumers have also been created (to test that the server uses - unique consumer tags). - - - - - - - - - - - Request exclusive consumer access, meaning only this consumer can access the - queue. - - - - - The client MAY NOT gain exclusive access to a queue that already has - active consumers. - - - Open two connections to a server, and in one connection declare a shared - (non-exclusive) queue and then consume from the queue. In the second - connection attempt to consume from the same queue using the exclusive - option. - - - - - - - - - A set of arguments for the consume. The syntax and semantics of these - arguments depends on the server implementation. - - - - - - - The server provides the client with a consumer tag, which is used by the client - for methods called on the consumer at a later stage. - - - - - Holds the consumer tag specified by the client or provided by the server. - - - - - - - - - This method cancels a consumer. This does not affect already delivered - messages, but it does mean the server will not send any more messages for - that consumer. The client may receive an arbitrary number of messages in - between sending the cancel method and receiving the cancel-ok reply. - - - - - If the queue does not exist the server MUST ignore the cancel method, so - long as the consumer tag is valid for that channel. - - - TODO. - - - - - - - - - - - - - This method confirms that the cancellation was completed. - - - - - - - - - - This method publishes a message to a specific exchange. The message will be routed - to queues as defined by the exchange configuration and distributed to any active - consumers when the transaction, if any, is committed. - - - - - - - - - - Specifies the name of the exchange to publish to. The exchange name can be - empty, meaning the default exchange. If the exchange name is specified, and that - exchange does not exist, the server will raise a channel exception. - - - - - The client MUST NOT attempt to publish a content to an exchange that - does not exist. - - - The client attempts to publish a content to a non-existent exchange. - - - - - The server MUST accept a blank exchange name to mean the default exchange. - - - The client declares a queue and binds it to a blank exchange name. - - - - - If the exchange was declared as an internal exchange, the server MUST raise - a channel exception with a reply code 403 (access refused). - - - TODO. - - - - - - The exchange MAY refuse basic content in which case it MUST raise a channel - exception with reply code 540 (not implemented). - - - TODO. - - - - - - - Specifies the routing key for the message. The routing key is used for routing - messages depending on the exchange configuration. - - - - - - This flag tells the server how to react if the message cannot be routed to a - queue. If this flag is set, the server will return an unroutable message with a - Return method. If this flag is zero, the server silently drops the message. - - - - - The server SHOULD implement the mandatory flag. - - - TODO. - - - - - - - This flag tells the server how to react if the message cannot be routed to a - queue consumer immediately. If this flag is set, the server will return an - undeliverable message with a Return method. If this flag is zero, the server - will queue the message, but with no guarantee that it will ever be consumed. - - - - - The server SHOULD implement the immediate flag. - - - TODO. - - - - - - - - This method returns an undeliverable message that was published with the "immediate" - flag set, or an unroutable message published with the "mandatory" flag set. The - reply code and text provide information about the reason that the message was - undeliverable. - - - - - - - - - - Specifies the name of the exchange that the message was originally published - to. May be empty, meaning the default exchange. - - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - - This method delivers a message to the client, via a consumer. In the asynchronous - message delivery model, the client starts a consumer using the Consume method, then - the server responds with Deliver methods as and when messages arrive for that - consumer. - - - - - The server SHOULD track the number of times a message has been delivered to - clients and when a message is redelivered a certain number of times - e.g. 5 - times - without being acknowledged, the server SHOULD consider the message to be - unprocessable (possibly causing client applications to abort), and move the - message to a dead letter queue. - - - TODO. - - - - - - - - - - - - Specifies the name of the exchange that the message was originally published to. - May be empty, indicating the default exchange. - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - This method provides a direct access to the messages in a queue using a synchronous - dialogue that is designed for specific types of application where synchronous - functionality is more important than performance. - - - - - - - - - - - Specifies the name of the queue to get a message from. - - - - - - - This method delivers a message to the client following a get method. A message - delivered by 'get-ok' must be acknowledged unless the no-ack option was set in the - get method. - - - - - - - - - Specifies the name of the exchange that the message was originally published to. - If empty, the message was published to the default exchange. - - - - - Specifies the routing key name specified when the message was published. - - - - - - - - This method tells the client that the queue has no messages available for the - client. - - - - - - - - - - - This method acknowledges one or more messages delivered via the Deliver or Get-Ok - methods. The client can ask to confirm a single message or a set of messages up to - and including a specific message. - - - - - - - - If set to 1, the delivery tag is treated as "up to and including", so that the - client can acknowledge multiple messages with a single method. If set to zero, - the delivery tag refers to a single message. If the multiple field is 1, and the - delivery tag is zero, tells the server to acknowledge all outstanding messages. - - - - The server MUST validate that a non-zero delivery-tag refers to a delivered - message, and raise a channel exception if this is not the case. On a transacted - channel, this check MUST be done immediately and not delayed until a Tx.Commit. - Specifically, a client MUST not acknowledge the same message more than once. - - - TODO. - - - - - - - - - - This method allows a client to reject a message. It can be used to interrupt and - cancel large incoming messages, or return untreatable messages to their original - queue. - - - - - The server SHOULD be capable of accepting and process the Reject method while - sending message content with a Deliver or Get-Ok method. I.e. the server should - read and process incoming methods while sending output frames. To cancel a - partially-send content, the server sends a content body frame of size 1 (i.e. - with no data except the frame-end octet). - - - - - - The server SHOULD interpret this method as meaning that the client is unable to - process the message at this time. - - - TODO. - - - - - - The client MUST NOT use this method as a means of selecting messages to process. - - - TODO. - - - - - - - - - - If requeue is true, the server will attempt to requeue the message. If requeue - is false or the requeue attempt fails the messages are discarded or dead-lettered. - - - - - The server MUST NOT deliver the message to the same client within the - context of the current channel. The recommended strategy is to attempt to - deliver the message to an alternative consumer, and if that is not possible, - to move the message to a dead-letter queue. The server MAY use more - sophisticated tracking to hold the message on the queue and redeliver it to - the same client at a later stage. - - - TODO. - - - - - - - - - - This method asks the server to redeliver all unacknowledged messages on a - specified channel. Zero or more messages may be redelivered. This method - is deprecated in favour of the synchronous Recover/Recover-Ok. - - - - The server MUST set the redelivered flag on all messages that are resent. - - - TODO. - - - - - - If this field is zero, the message will be redelivered to the original - recipient. If this bit is 1, the server will attempt to requeue the message, - potentially then delivering it to an alternative subscriber. - - - - - - - - - This method asks the server to redeliver all unacknowledged messages on a - specified channel. Zero or more messages may be redelivered. This method - replaces the asynchronous Recover. - - - - The server MUST set the redelivered flag on all messages that are resent. - - - TODO. - - - - - - If this field is zero, the message will be redelivered to the original - recipient. If this bit is 1, the server will attempt to requeue the message, - potentially then delivering it to an alternative subscriber. - - - - - - - This method acknowledges a Basic.Recover method. - - - - - - - - - - The Tx class allows publish and ack operations to be batched into atomic - units of work. The intention is that all publish and ack requests issued - within a transaction will complete successfully or none of them will. - Servers SHOULD implement atomic transactions at least where all publish - or ack requests affect a single queue. Transactions that cover multiple - queues may be non-atomic, given that queues can be created and destroyed - asynchronously, and such events do not form part of any transaction. - Further, the behaviour of transactions with respect to the immediate and - mandatory flags on Basic.Publish methods is not defined. - - - - - Applications MUST NOT rely on the atomicity of transactions that - affect more than one queue. - - - - - Applications MUST NOT rely on the behaviour of transactions that - include messages published with the immediate option. - - - - - Applications MUST NOT rely on the behaviour of transactions that - include messages published with the mandatory option. - - - - - tx = C:SELECT S:SELECT-OK - / C:COMMIT S:COMMIT-OK - / C:ROLLBACK S:ROLLBACK-OK - - - - - - - - - - This method sets the channel to use standard transactions. The client must use this - method at least once on a channel before using the Commit or Rollback methods. - - - - - - - - This method confirms to the client that the channel was successfully set to use - standard transactions. - - - - - - - - - This method commits all message publications and acknowledgments performed in - the current transaction. A new transaction starts immediately after a commit. - - - - - - - The client MUST NOT use the Commit method on non-transacted channels. - - - The client opens a channel and then uses Tx.Commit. - - - - - - - This method confirms to the client that the commit succeeded. Note that if a commit - fails, the server raises a channel exception. - - - - - - - - - This method abandons all message publications and acknowledgments performed in - the current transaction. A new transaction starts immediately after a rollback. - Note that unacked messages will not be automatically redelivered by rollback; - if that is required an explicit recover call should be issued. - - - - - - - The client MUST NOT use the Rollback method on non-transacted channels. - - - The client opens a channel and then uses Tx.Rollback. - - - - - - - This method confirms to the client that the rollback succeeded. Note that if an - rollback fails, the server raises a channel exception. - - - - - - + + + + + + + + + + + + + + + + + + + + + + + Indicates that the method completed successfully. This reply code is + reserved for future use - the current protocol design does not use positive + confirmation and reply codes are sent only in case of an error. + + + + + + The client attempted to transfer content larger than the server could accept + at the present time. The client may retry at a later time. + + + + + + When the exchange cannot deliver to a consumer when the immediate flag is + set. As a result of pending data on the queue or the absence of any + consumers of the queue. + + + + + + An operator intervened to close the connection for some reason. The client + may retry at some later date. + + + + + + The client tried to work with an unknown virtual host. + + + + + + The client attempted to work with a server entity to which it has no + access due to security settings. + + + + + + The client attempted to work with a server entity that does not exist. + + + + + + The client attempted to work with a server entity to which it has no + access because another client is working with it. + + + + + + The client requested a method that was not allowed because some precondition + failed. + + + + + + The sender sent a malformed frame that the recipient could not decode. + This strongly implies a programming error in the sending peer. + + + + + + The sender sent a frame that contained illegal values for one or more + fields. This strongly implies a programming error in the sending peer. + + + + + + The client sent an invalid sequence of frames, attempting to perform an + operation that was considered invalid by the server. This usually implies + a programming error in the client. + + + + + + The client attempted to work with a channel that had not been correctly + opened. This most likely indicates a fault in the client layer. + + + + + + The peer sent a frame that was not expected, usually in the context of + a content header and body. This strongly indicates a fault in the peer's + content processing. + + + + + + The server could not complete the method because it lacked sufficient + resources. This may be due to the client creating too many of some type + of entity. + + + + + + The client tried to work with some entity in a manner that is prohibited + by the server, due to security settings or by some other criteria. + + + + + + The client tried to use functionality that is not implemented in the + server. + + + + + + The server could not complete the method because of an internal error. + The server may require intervention by an operator in order to resume + normal operations. + + + + + + + + + + Identifier for the consumer, valid within the current channel. + + + + + + The server-assigned and channel-specific delivery tag + + + + The delivery tag is valid only within the channel from which the message was + received. I.e. a client MUST NOT receive a message on one channel and then + acknowledge it on another. + + + + + The server MUST NOT use a zero value for delivery tags. Zero is reserved + for client use, meaning "all messages so far received". + + + + + + + The exchange name is a client-selected string that identifies the exchange for + publish methods. + + + + + + + + + + If this field is set the server does not expect acknowledgements for + messages. That is, when a message is delivered to the client the server + assumes the delivery will succeed and immediately dequeues it. This + functionality may increase performance but at the cost of reliability. + Messages can get lost if a client dies before they are delivered to the + application. + + + + + + If the no-local field is set the server will not send messages to the connection that + published them. + + + + + + If set, the server will not respond to the method. The client should not wait + for a reply method. If the server could not complete the method it will raise a + channel or connection exception. + + + + + + Unconstrained. + + + + + + + + This table provides a set of peer properties, used for identification, debugging, + and general information. + + + + + + The queue name identifies the queue within the vhost. In methods where the queue + name may be blank, and that has no specific significance, this refers to the + 'current' queue for the channel, meaning the last queue that the client declared + on the channel. If the client did not declare a queue, and the method needs a + queue name, this will result in a 502 (syntax error) channel exception. + + + + + + + + This indicates that the message has been previously delivered to this or + another client. + + + + The server SHOULD try to signal redelivered messages when it can. When + redelivering a message that was not successfully acknowledged, the server + SHOULD deliver it to the original client if possible. + + + Declare a shared queue and publish a message to the queue. Consume the + message using explicit acknowledgements, but do not acknowledge the + message. Close the connection, reconnect, and consume from the queue + again. The message should arrive with the redelivered flag set. + + + + + The client MUST NOT rely on the redelivered field but should take it as a + hint that the message may already have been processed. A fully robust + client must be able to track duplicate received messages on non-transacted, + and locally-transacted channels. + + + + + + + The number of messages in the queue, which will be zero for newly-declared + queues. This is the number of messages present in the queue, and committed + if the channel on which they were published is transacted, that are not + waiting acknowledgement. + + + + + + The reply code. The AMQ reply codes are defined as constants at the start + of this formal specification. + + + + + + + The localised reply text. This text can be logged as an aid to resolving + issues. + + + + + + + + + + + + + + + + + + + + The connection class provides methods for a client to establish a network connection to + a server, and for both peers to operate the connection thereafter. + + + + connection = open-connection *use-connection close-connection + open-connection = C:protocol-header + S:START C:START-OK + *challenge + S:TUNE C:TUNE-OK + C:OPEN S:OPEN-OK + challenge = S:SECURE C:SECURE-OK + use-connection = *channel + close-connection = C:CLOSE S:CLOSE-OK + / S:CLOSE C:CLOSE-OK + + + + + + + + + + This method starts the connection negotiation process by telling the client the + protocol version that the server proposes, along with a list of security mechanisms + which the client can use for authentication. + + + + + If the server cannot support the protocol specified in the protocol header, + it MUST respond with a valid protocol header and then close the socket + connection. + + + The client sends a protocol header containing an invalid protocol name. + The server MUST respond by sending a valid protocol header and then closing + the connection. + + + + + The server MUST provide a protocol version that is lower than or equal to + that requested by the client in the protocol header. + + + The client requests a protocol version that is higher than any valid + implementation, e.g. 2.0. The server must respond with a protocol header + indicating its supported protocol version, e.g. 1.0. + + + + + If the client cannot handle the protocol version suggested by the server + it MUST close the socket connection without sending any further data. + + + The server sends a protocol version that is lower than any valid + implementation, e.g. 0.1. The client must respond by closing the + connection without sending any further data. + + + + + + + + + The major version number can take any value from 0 to 99 as defined in the + AMQP specification. + + + + + + The minor version number can take any value from 0 to 99 as defined in the + AMQP specification. + + + + + + + The properties SHOULD contain at least these fields: "host", specifying the + server host name or address, "product", giving the name of the server product, + "version", giving the name of the server version, "platform", giving the name + of the operating system, "copyright", if appropriate, and "information", giving + other general information. + + + Client connects to server and inspects the server properties. It checks for + the presence of the required fields. + + + + + + + A list of the security mechanisms that the server supports, delimited by spaces. + + + + + + + A list of the message locales that the server supports, delimited by spaces. The + locale defines the language in which the server will send reply texts. + + + + The server MUST support at least the en_US locale. + + + Client connects to server and inspects the locales field. It checks for + the presence of the required locale(s). + + + + + + + + + This method selects a SASL security mechanism. + + + + + + + + + The properties SHOULD contain at least these fields: "product", giving the name + of the client product, "version", giving the name of the client version, "platform", + giving the name of the operating system, "copyright", if appropriate, and + "information", giving other general information. + + + + + + + A single security mechanisms selected by the client, which must be one of those + specified by the server. + + + + The client SHOULD authenticate using the highest-level security profile it + can handle from the list provided by the server. + + + + + If the mechanism field does not contain one of the security mechanisms + proposed by the server in the Start method, the server MUST close the + connection without sending any further data. + + + Client connects to server and sends an invalid security mechanism. The + server must respond by closing the connection (a socket close, with no + connection close negotiation). + + + + + + + + A block of opaque data passed to the security mechanism. The contents of this + data are defined by the SASL security mechanism. + + + + + + + A single message locale selected by the client, which must be one of those + specified by the server. + + + + + + + + + + The SASL protocol works by exchanging challenges and responses until both peers have + received sufficient information to authenticate each other. This method challenges + the client to provide more information. + + + + + + + + Challenge information, a block of opaque binary data passed to the security + mechanism. + + + + + + + This method attempts to authenticate, passing a block of SASL data for the security + mechanism at the server side. + + + + + + + A block of opaque data passed to the security mechanism. The contents of this + data are defined by the SASL security mechanism. + + + + + + + + + + This method proposes a set of connection configuration values to the client. The + client can accept and/or adjust these. + + + + + + + + + Specifies highest channel number that the server permits. Usable channel numbers + are in the range 1..channel-max. Zero indicates no specified limit. + + + + + + The largest frame size that the server proposes for the connection, including + frame header and end-byte. The client can negotiate a lower value. Zero means + that the server does not impose any specific limit but may reject very large + frames if it cannot allocate resources for them. + + + + Until the frame-max has been negotiated, both peers MUST accept frames of up + to frame-min-size octets large, and the minimum negotiated value for frame-max + is also frame-min-size. + + + Client connects to server and sends a large properties field, creating a frame + of frame-min-size octets. The server must accept this frame. + + + + + + + The delay, in seconds, of the connection heartbeat that the server wants. + Zero means the server does not want a heartbeat. + + + + + + + This method sends the client's connection tuning parameters to the server. + Certain fields are negotiated, others provide capability information. + + + + + + + The maximum total number of channels that the client will use per connection. + + + + If the client specifies a channel max that is higher than the value provided + by the server, the server MUST close the connection without attempting a + negotiated close. The server may report the error in some fashion to assist + implementors. + + + + + + + + + The largest frame size that the client and server will use for the connection. + Zero means that the client does not impose any specific limit but may reject + very large frames if it cannot allocate resources for them. Note that the + frame-max limit applies principally to content frames, where large contents can + be broken into frames of arbitrary size. + + + + Until the frame-max has been negotiated, both peers MUST accept frames of up + to frame-min-size octets large, and the minimum negotiated value for frame-max + is also frame-min-size. + + + + + If the client specifies a frame max that is higher than the value provided + by the server, the server MUST close the connection without attempting a + negotiated close. The server may report the error in some fashion to assist + implementors. + + + + + + + The delay, in seconds, of the connection heartbeat that the client wants. Zero + means the client does not want a heartbeat. + + + + + + + + + This method opens a connection to a virtual host, which is a collection of + resources, and acts to separate multiple application domains within a server. + The server may apply arbitrary limits per virtual host, such as the number + of each type of entity that may be used, per connection and/or in total. + + + + + + + + The name of the virtual host to work with. + + + + If the server supports multiple virtual hosts, it MUST enforce a full + separation of exchanges, queues, and all associated entities per virtual + host. An application, connected to a specific virtual host, MUST NOT be able + to access resources of another virtual host. + + + + + The server SHOULD verify that the client has permission to access the + specified virtual host. + + + + + + + + + + + + This method signals to the client that the connection is ready for use. + + + + + + + + + + + This method indicates that the sender wants to close the connection. This may be + due to internal conditions (e.g. a forced shut-down) or due to an error handling + a specific method, i.e. an exception. When a close is due to an exception, the + sender provides the class and method id of the method which caused the exception. + + + + After sending this method, any received methods except Close and Close-OK MUST + be discarded. The response to receiving a Close after sending Close must be to + send Close-Ok. + + + + + + + + + + + + + When the close is provoked by a method exception, this is the class of the + method. + + + + + + When the close is provoked by a method exception, this is the ID of the method. + + + + + + + This method confirms a Connection.Close method and tells the recipient that it is + safe to release resources for the connection and close the socket. + + + + A peer that detects a socket closure without having received a Close-Ok + handshake method SHOULD log the error. + + + + + + + + + + + + The channel class provides methods for a client to establish a channel to a + server and for both peers to operate the channel thereafter. + + + + channel = open-channel *use-channel close-channel + open-channel = C:OPEN S:OPEN-OK + use-channel = C:FLOW S:FLOW-OK + / S:FLOW C:FLOW-OK + / functional-class + close-channel = C:CLOSE S:CLOSE-OK + / S:CLOSE C:CLOSE-OK + + + + + + + + + + This method opens a channel to the server. + + + + The client MUST NOT use this method on an already-opened channel. + + + Client opens a channel and then reopens the same channel. + + + + + + + + + + + This method signals to the client that the channel is ready for use. + + + + + + + + + + + This method asks the peer to pause or restart the flow of content data sent by + a consumer. This is a simple flow-control mechanism that a peer can use to avoid + overflowing its queues or otherwise finding itself receiving more messages than + it can process. Note that this method is not intended for window control. It does + not affect contents returned by Basic.Get-Ok methods. + + + + + When a new channel is opened, it is active (flow is active). Some applications + assume that channels are inactive until started. To emulate this behaviour a + client MAY open the channel, then pause it. + + + + + + When sending content frames, a peer SHOULD monitor the channel for incoming + methods and respond to a Channel.Flow as rapidly as possible. + + + + + + A peer MAY use the Channel.Flow method to throttle incoming content data for + internal reasons, for example, when exchanging data over a slower connection. + + + + + + The peer that requests a Channel.Flow method MAY disconnect and/or ban a peer + that does not respect the request. This is to prevent badly-behaved clients + from overwhelming a server. + + + + + + + + + + + If 1, the peer starts sending content frames. If 0, the peer stops sending + content frames. + + + + + + + Confirms to the peer that a flow command was received and processed. + + + + + + Confirms the setting of the processed flow method: 1 means the peer will start + sending or continue to send content frames; 0 means it will not. + + + + + + + + + This method indicates that the sender wants to close the channel. This may be due to + internal conditions (e.g. a forced shut-down) or due to an error handling a specific + method, i.e. an exception. When a close is due to an exception, the sender provides + the class and method id of the method which caused the exception. + + + + After sending this method, any received methods except Close and Close-OK MUST + be discarded. The response to receiving a Close after sending Close must be to + send Close-Ok. + + + + + + + + + + + + + When the close is provoked by a method exception, this is the class of the + method. + + + + + + When the close is provoked by a method exception, this is the ID of the method. + + + + + + + This method confirms a Channel.Close method and tells the recipient that it is safe + to release resources for the channel. + + + + A peer that detects a socket closure without having received a Channel.Close-Ok + handshake method SHOULD log the error. + + + + + + + + + + + + Exchanges match and distribute messages across queues. Exchanges can be configured in + the server or declared at runtime. + + + + exchange = C:DECLARE S:DECLARE-OK + / C:DELETE S:DELETE-OK + + + + + + + + The server MUST implement these standard exchange types: fanout, direct. + + + Client attempts to declare an exchange with each of these standard types. + + + + + The server SHOULD implement these standard exchange types: topic, headers. + + + Client attempts to declare an exchange with each of these standard types. + + + + + The server MUST, in each virtual host, pre-declare an exchange instance + for each standard exchange type that it implements, where the name of the + exchange instance, if defined, is "amq." followed by the exchange type name. + + + The server MUST, in each virtual host, pre-declare at least two direct + exchange instances: one named "amq.direct", the other with no public name + that serves as a default exchange for Publish methods. + + + Client declares a temporary queue and attempts to bind to each required + exchange instance ("amq.fanout", "amq.direct", "amq.topic", and "amq.headers" + if those types are defined). + + + + + The server MUST pre-declare a direct exchange with no public name to act as + the default exchange for content Publish methods and for default queue bindings. + + + Client checks that the default exchange is active by specifying a queue + binding with no exchange name, and publishing a message with a suitable + routing key but without specifying the exchange name, then ensuring that + the message arrives in the queue correctly. + + + + + The server MUST NOT allow clients to access the default exchange except + by specifying an empty exchange name in the Queue.Bind and content Publish + methods. + + + + + The server MAY implement other exchange types as wanted. + + + + + + + + This method creates an exchange if it does not already exist, and if the exchange + exists, verifies that it is of the correct and expected class. + + + + The server SHOULD support a minimum of 16 exchanges per virtual host and + ideally, impose no limit except as defined by available resources. + + + The client declares as many exchanges as it can until the server reports + an error; the number of exchanges successfully declared must be at least + sixteen. + + + + + + + + + + + + + Exchange names starting with "amq." are reserved for pre-declared and + standardised exchanges. The client MAY declare an exchange starting with + "amq." if the passive option is set, or the exchange already exists. + + + The client attempts to declare a non-existing exchange starting with + "amq." and with the passive option set to zero. + + + + + The exchange name consists of a non-empty sequence of these characters: + letters, digits, hyphen, underscore, period, or colon. + + + The client attempts to declare an exchange with an illegal name. + + + + + + + + Each exchange belongs to one of a set of exchange types implemented by the + server. The exchange types define the functionality of the exchange - i.e. how + messages are routed through it. It is not valid or meaningful to attempt to + change the type of an existing exchange. + + + + Exchanges cannot be redeclared with different types. The client MUST not + attempt to redeclare an existing exchange with a different type than used + in the original Exchange.Declare method. + + + TODO. + + + + + The client MUST NOT attempt to declare an exchange with a type that the + server does not support. + + + TODO. + + + + + + + If set, the server will reply with Declare-Ok if the exchange already + exists with the same name, and raise an error if not. The client can + use this to check whether an exchange exists without modifying the + server state. When set, all other method fields except name and no-wait + are ignored. A declare with both passive and no-wait has no effect. + Arguments are compared for semantic equivalence. + + + + If set, and the exchange does not already exist, the server MUST + raise a channel exception with reply code 404 (not found). + + + TODO. + + + + + If not set and the exchange exists, the server MUST check that the + existing exchange has the same values for type, durable, and arguments + fields. The server MUST respond with Declare-Ok if the requested + exchange matches these fields, and MUST raise a channel exception if + not. + + + TODO. + + + + + + + If set when creating a new exchange, the exchange will be marked as durable. + Durable exchanges remain active when a server restarts. Non-durable exchanges + (transient exchanges) are purged if/when a server restarts. + + + + The server MUST support both durable and transient exchanges. + + + TODO. + + + + + + + + + + + + + A set of arguments for the declaration. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + This method confirms a Declare method and confirms the name of the exchange, + essential for automatically-named exchanges. + + + + + + + + + This method deletes an exchange. When an exchange is deleted all queue bindings on + the exchange are cancelled. + + + + + + + + + + + + The client MUST NOT attempt to delete an exchange that does not exist. + + + + + + + + If set, the server will only delete the exchange if it has no queue bindings. If + the exchange has queue bindings the server does not delete it but raises a + channel exception instead. + + + + The server MUST NOT delete an exchange that has bindings on it, if the if-unused + field is true. + + + The client declares an exchange, binds a queue to it, then tries to delete it + setting if-unused to true. + + + + + + + + + This method confirms the deletion of an exchange. + + + + + + + + + Queues store and forward messages. Queues can be configured in the server or created at + runtime. Queues must be attached to at least one exchange in order to receive messages + from publishers. + + + + queue = C:DECLARE S:DECLARE-OK + / C:BIND S:BIND-OK + / C:UNBIND S:UNBIND-OK + / C:PURGE S:PURGE-OK + / C:DELETE S:DELETE-OK + + + + + + + + + + This method creates or checks a queue. When creating a new queue the client can + specify various properties that control the durability of the queue and its + contents, and the level of sharing for the queue. + + + + + The server MUST create a default binding for a newly-declared queue to the + default exchange, which is an exchange of type 'direct' and use the queue + name as the routing key. + + + Client declares a new queue, and then without explicitly binding it to an + exchange, attempts to send a message through the default exchange binding, + i.e. publish a message to the empty exchange, with the queue name as routing + key. + + + + + + The server SHOULD support a minimum of 256 queues per virtual host and ideally, + impose no limit except as defined by available resources. + + + Client attempts to declare as many queues as it can until the server reports + an error. The resulting count must at least be 256. + + + + + + + + + + + + + The queue name MAY be empty, in which case the server MUST create a new + queue with a unique generated name and return this to the client in the + Declare-Ok method. + + + Client attempts to declare several queues with an empty name. The client then + verifies that the server-assigned names are unique and different. + + + + + Queue names starting with "amq." are reserved for pre-declared and + standardised queues. The client MAY declare a queue starting with + "amq." if the passive option is set, or the queue already exists. + + + The client attempts to declare a non-existing queue starting with + "amq." and with the passive option set to zero. + + + + + The queue name can be empty, or a sequence of these characters: + letters, digits, hyphen, underscore, period, or colon. + + + The client attempts to declare a queue with an illegal name. + + + + + + + If set, the server will reply with Declare-Ok if the queue already + exists with the same name, and raise an error if not. The client can + use this to check whether a queue exists without modifying the + server state. When set, all other method fields except name and no-wait + are ignored. A declare with both passive and no-wait has no effect. + Arguments are compared for semantic equivalence. + + + + The client MAY ask the server to assert that a queue exists without + creating the queue if not. If the queue does not exist, the server + treats this as a failure. + + + Client declares an existing queue with the passive option and expects + the server to respond with a declare-ok. Client then attempts to declare + a non-existent queue with the passive option, and the server must close + the channel with the correct reply-code. + + + + + If not set and the queue exists, the server MUST check that the + existing queue has the same values for durable, exclusive, auto-delete, + and arguments fields. The server MUST respond with Declare-Ok if the + requested queue matches these fields, and MUST raise a channel exception + if not. + + + TODO. + + + + + + + If set when creating a new queue, the queue will be marked as durable. Durable + queues remain active when a server restarts. Non-durable queues (transient + queues) are purged if/when a server restarts. Note that durable queues do not + necessarily hold persistent messages, although it does not make sense to send + persistent messages to a transient queue. + + + + The server MUST recreate the durable queue after a restart. + + + Client declares a durable queue. The server is then restarted. The client + then attempts to send a message to the queue. The message should be successfully + delivered. + + + + + The server MUST support both durable and transient queues. + + A client declares two named queues, one durable and one transient. + + + + + + + Exclusive queues may only be accessed by the current connection, and are + deleted when that connection closes. Passive declaration of an exclusive + queue by other connections are not allowed. + + + + + The server MUST support both exclusive (private) and non-exclusive (shared) + queues. + + + A client declares two named queues, one exclusive and one non-exclusive. + + + + + + The client MAY NOT attempt to use a queue that was declared as exclusive + by another still-open connection. + + + One client declares an exclusive queue. A second client on a different + connection attempts to declare, bind, consume, purge, delete, or declare + a queue of the same name. + + + + + + + If set, the queue is deleted when all consumers have finished using it. The last + consumer can be cancelled either explicitly or because its channel is closed. If + there was no consumer ever on the queue, it won't be deleted. Applications can + explicitly delete auto-delete queues using the Delete method as normal. + + + + + The server MUST ignore the auto-delete field if the queue already exists. + + + Client declares two named queues, one as auto-delete and one explicit-delete. + Client then attempts to declare the two queues using the same names again, + but reversing the value of the auto-delete field in each case. Verify that the + queues still exist with the original auto-delete flag values. + + + + + + + + + A set of arguments for the declaration. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + This method confirms a Declare method and confirms the name of the queue, essential + for automatically-named queues. + + + + + + + Reports the name of the queue. If the server generated a queue name, this field + contains that name. + + + + + + + + + Reports the number of active consumers for the queue. Note that consumers can + suspend activity (Channel.Flow) in which case they do not appear in this count. + + + + + + + + + This method binds a queue to an exchange. Until a queue is bound it will not + receive any messages. In a classic messaging model, store-and-forward queues + are bound to a direct exchange and subscription queues are bound to a topic + exchange. + + + + + A server MUST allow ignore duplicate bindings - that is, two or more bind + methods for a specific queue, with identical arguments - without treating these + as an error. + + + A client binds a named queue to an exchange. The client then repeats the bind + (with identical arguments). + + + + + + A server MUST not deliver the same message more than once to a queue, even if + the queue has multiple bindings that match the message. + + + A client declares a named queue and binds it using multiple bindings to the + amq.topic exchange. The client then publishes a message that matches all its + bindings. + + + + + + The server MUST allow a durable queue to bind to a transient exchange. + + + A client declares a transient exchange. The client then declares a named durable + queue and then attempts to bind the transient exchange to the durable queue. + + + + + + Bindings of durable queues to durable exchanges are automatically durable + and the server MUST restore such bindings after a server restart. + + + A server declares a named durable queue and binds it to a durable exchange. The + server is restarted. The client then attempts to use the queue/exchange combination. + + + + + + The server SHOULD support at least 4 bindings per queue, and ideally, impose no + limit except as defined by available resources. + + + A client declares a named queue and attempts to bind it to 4 different + exchanges. + + + + + + + + + + + + Specifies the name of the queue to bind. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to bind an unnamed queue. + + + + + The client MUST NOT attempt to bind a queue that does not exist. + + + The client attempts to bind a non-existent queue. + + + + + + + + A client MUST NOT be allowed to bind a queue to a non-existent exchange. + + + A client attempts to bind an named queue to a undeclared exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + + + Specifies the routing key for the binding. The routing key is used for routing + messages depending on the exchange configuration. Not all exchanges use a + routing key - refer to the specific exchange documentation. If the queue name + is empty, the server uses the last queue declared on the channel. If the + routing key is also empty, the server uses this queue name for the routing + key as well. If the queue name is provided but the routing key is empty, the + server does the binding with that empty routing key. The meaning of empty + routing keys depends on the exchange implementation. + + + + If a message queue binds to a direct exchange using routing key K and a + publisher sends the exchange a message with routing key R, then the message + MUST be passed to the message queue if K = R. + + + + + + + + + A set of arguments for the binding. The syntax and semantics of these arguments + depends on the exchange class. + + + + + + This method confirms that the bind was successful. + + + + + + + + This method unbinds a queue from an exchange. + + If a unbind fails, the server MUST raise a connection exception. + + + + + + + + + Specifies the name of the queue to unbind. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to unbind an unnamed queue. + + + + + The client MUST NOT attempt to unbind a queue that does not exist. + + + The client attempts to unbind a non-existent queue. + + + + + + The name of the exchange to unbind from. + + + The client MUST NOT attempt to unbind a queue from an exchange that + does not exist. + + + The client attempts to unbind a queue from a non-existent exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + + Specifies the routing key of the binding to unbind. + + + + Specifies the arguments of the binding to unbind. + + + + + This method confirms that the unbind was successful. + + + + + + + + This method removes all messages from a queue which are not awaiting + acknowledgment. + + + + + The server MUST NOT purge messages that have already been sent to a client + but not yet acknowledged. + + + + + + The server MAY implement a purge queue or log that allows system administrators + to recover accidentally-purged messages. The server SHOULD NOT keep purged + messages in the same storage spaces as the live messages since the volumes of + purged messages may get very large. + + + + + + + + + + + + Specifies the name of the queue to purge. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to purge an unnamed queue. + + + + + The client MUST NOT attempt to purge a queue that does not exist. + + + The client attempts to purge a non-existent queue. + + + + + + + + + This method confirms the purge of a queue. + + + + + + Reports the number of messages purged. + + + + + + + + + This method deletes a queue. When a queue is deleted any pending messages are sent + to a dead-letter queue if this is defined in the server configuration, and all + consumers on the queue are cancelled. + + + + + The server SHOULD use a dead-letter queue to hold messages that were pending on + a deleted queue, and MAY provide facilities for a system administrator to move + these messages back to an active queue. + + + + + + + + + + + + Specifies the name of the queue to delete. + + + The client MUST either specify a queue name or have previously declared a + queue on the same channel + + + The client opens a channel and attempts to delete an unnamed queue. + + + + + The client MUST NOT attempt to delete a queue that does not exist. + + + The client attempts to delete a non-existent queue. + + + + + + + If set, the server will only delete the queue if it has no consumers. If the + queue has consumers the server does does not delete it but raises a channel + exception instead. + + + + The server MUST NOT delete a queue that has consumers on it, if the if-unused + field is true. + + + The client declares a queue, and consumes from it, then tries to delete it + setting if-unused to true. + + + + + + + If set, the server will only delete the queue if it has no messages. + + + + The server MUST NOT delete a queue that has messages on it, if the + if-empty field is true. + + + The client declares a queue, binds it and publishes some messages into it, + then tries to delete it setting if-empty to true. + + + + + + + + + This method confirms the deletion of a queue. + + + + + Reports the number of messages deleted. + + + + + + + + + The Basic class provides methods that support an industry-standard messaging model. + + + + basic = C:QOS S:QOS-OK + / C:CONSUME S:CONSUME-OK + / C:CANCEL S:CANCEL-OK + / C:PUBLISH content + / S:RETURN content + / S:DELIVER content + / C:GET ( S:GET-OK content / S:GET-EMPTY ) + / C:ACK + / C:REJECT + / C:RECOVER-ASYNC + / C:RECOVER S:RECOVER-OK + + + + + + + + The server SHOULD respect the persistent property of basic messages and + SHOULD make a best-effort to hold persistent basic messages on a reliable + storage mechanism. + + + Send a persistent message to queue, stop server, restart server and then + verify whether message is still present. Assumes that queues are durable. + Persistence without durable queues makes no sense. + + + + + + The server MUST NOT discard a persistent basic message in case of a queue + overflow. + + + Declare a queue overflow situation with persistent messages and verify that + messages do not get lost (presumably the server will write them to disk). + + + + + + The server MAY use the Channel.Flow method to slow or stop a basic message + publisher when necessary. + + + Declare a queue overflow situation with non-persistent messages and verify + whether the server responds with Channel.Flow or not. Repeat with persistent + messages. + + + + + + The server MAY overflow non-persistent basic messages to persistent + storage. + + + + + + + The server MAY discard or dead-letter non-persistent basic messages on a + priority basis if the queue size exceeds some configured limit. + + + + + + + The server MUST implement at least 2 priority levels for basic messages, + where priorities 0-4 and 5-9 are treated as two distinct levels. + + + Send a number of priority 0 messages to a queue. Send one priority 9 + message. Consume messages from the queue and verify that the first message + received was priority 9. + + + + + + The server MAY implement up to 10 priority levels. + + + Send a number of messages with mixed priorities to a queue, so that all + priority values from 0 to 9 are exercised. A good scenario would be ten + messages in low-to-high priority. Consume from queue and verify how many + priority levels emerge. + + + + + + The server MUST deliver messages of the same priority in order irrespective of + their individual persistence. + + + Send a set of messages with the same priority but different persistence + settings to a queue. Consume and verify that messages arrive in same order + as originally published. + + + + + + The server MUST support un-acknowledged delivery of Basic content, i.e. + consumers with the no-ack field set to TRUE. + + + + + + The server MUST support explicitly acknowledged delivery of Basic content, + i.e. consumers with the no-ack field set to FALSE. + + + Declare a queue and a consumer using explicit acknowledgements. Publish a + set of messages to the queue. Consume the messages but acknowledge only + half of them. Disconnect and reconnect, and consume from the queue. + Verify that the remaining messages are received. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method requests a specific quality of service. The QoS can be specified for the + current channel or for all channels on the connection. The particular properties and + semantics of a qos method always depend on the content class semantics. Though the + qos method could in principle apply to both peers, it is currently meaningful only + for the server. + + + + + + + + The client can request that messages be sent in advance so that when the client + finishes processing a message, the following message is already held locally, + rather than needing to be sent down the channel. Prefetching gives a performance + improvement. This field specifies the prefetch window size in octets. The server + will send a message in advance if it is equal to or smaller in size than the + available prefetch size (and also falls into other prefetch limits). May be set + to zero, meaning "no specific limit", although other prefetch limits may still + apply. The prefetch-size is ignored if the no-ack option is set. + + + + The server MUST ignore this setting when the client is not processing any + messages - i.e. the prefetch size does not limit the transfer of single + messages to a client, only the sending in advance of more messages while + the client still has one or more unacknowledged messages. + + + Define a QoS prefetch-size limit and send a single message that exceeds + that limit. Verify that the message arrives correctly. + + + + + + + Specifies a prefetch window in terms of whole messages. This field may be used + in combination with the prefetch-size field; a message will only be sent in + advance if both prefetch windows (and those at the channel and connection level) + allow it. The prefetch-count is ignored if the no-ack option is set. + + + + The server may send less data in advance than allowed by the client's + specified prefetch windows but it MUST NOT send more. + + + Define a QoS prefetch-size limit and a prefetch-count limit greater than + one. Send multiple messages that exceed the prefetch size. Verify that + no more than one message arrives at once. + + + + + + + By default the QoS settings apply to the current channel only. If this field is + set, they are applied to the entire connection. + + + + + + + This method tells the client that the requested QoS levels could be handled by the + server. The requested QoS applies to all active consumers until a new QoS is + defined. + + + + + + + + + This method asks the server to start a "consumer", which is a transient request for + messages from a specific queue. Consumers last as long as the channel they were + declared on, or until the client cancels them. + + + + + The server SHOULD support at least 16 consumers per queue, and ideally, impose + no limit except as defined by available resources. + + + Declare a queue and create consumers on that queue until the server closes the + connection. Verify that the number of consumers created was at least sixteen + and report the total number. + + + + + + + + + + + Specifies the name of the queue to consume from. + + + + + Specifies the identifier for the consumer. The consumer tag is local to a + channel, so two clients can use the same consumer tags. If this field is + empty the server will generate a unique tag. + + + + The client MUST NOT specify a tag that refers to an existing consumer. + + + Attempt to create two consumers with the same non-empty tag, on the + same channel. + + + + + The consumer tag is valid only within the channel from which the + consumer was created. I.e. a client MUST NOT create a consumer in one + channel and then use it in another. + + + Attempt to create a consumer in one channel, then use in another channel, + in which consumers have also been created (to test that the server uses + unique consumer tags). + + + + + + + + + + + Request exclusive consumer access, meaning only this consumer can access the + queue. + + + + + The client MAY NOT gain exclusive access to a queue that already has + active consumers. + + + Open two connections to a server, and in one connection declare a shared + (non-exclusive) queue and then consume from the queue. In the second + connection attempt to consume from the same queue using the exclusive + option. + + + + + + + + + A set of arguments for the consume. The syntax and semantics of these + arguments depends on the server implementation. + + + + + + + The server provides the client with a consumer tag, which is used by the client + for methods called on the consumer at a later stage. + + + + + Holds the consumer tag specified by the client or provided by the server. + + + + + + + + + This method cancels a consumer. This does not affect already delivered + messages, but it does mean the server will not send any more messages for + that consumer. The client may receive an arbitrary number of messages in + between sending the cancel method and receiving the cancel-ok reply. + + + + + If the queue does not exist the server MUST ignore the cancel method, so + long as the consumer tag is valid for that channel. + + + TODO. + + + + + + + + + + + + + This method confirms that the cancellation was completed. + + + + + + + + + + This method publishes a message to a specific exchange. The message will be routed + to queues as defined by the exchange configuration and distributed to any active + consumers when the transaction, if any, is committed. + + + + + + + + + + Specifies the name of the exchange to publish to. The exchange name can be + empty, meaning the default exchange. If the exchange name is specified, and that + exchange does not exist, the server will raise a channel exception. + + + + + The client MUST NOT attempt to publish a content to an exchange that + does not exist. + + + The client attempts to publish a content to a non-existent exchange. + + + + + The server MUST accept a blank exchange name to mean the default exchange. + + + The client declares a queue and binds it to a blank exchange name. + + + + + If the exchange was declared as an internal exchange, the server MUST raise + a channel exception with a reply code 403 (access refused). + + + TODO. + + + + + + The exchange MAY refuse basic content in which case it MUST raise a channel + exception with reply code 540 (not implemented). + + + TODO. + + + + + + + Specifies the routing key for the message. The routing key is used for routing + messages depending on the exchange configuration. + + + + + + This flag tells the server how to react if the message cannot be routed to a + queue. If this flag is set, the server will return an unroutable message with a + Return method. If this flag is zero, the server silently drops the message. + + + + + The server SHOULD implement the mandatory flag. + + + TODO. + + + + + + + This flag tells the server how to react if the message cannot be routed to a + queue consumer immediately. If this flag is set, the server will return an + undeliverable message with a Return method. If this flag is zero, the server + will queue the message, but with no guarantee that it will ever be consumed. + + + + + The server SHOULD implement the immediate flag. + + + TODO. + + + + + + + + This method returns an undeliverable message that was published with the "immediate" + flag set, or an unroutable message published with the "mandatory" flag set. The + reply code and text provide information about the reason that the message was + undeliverable. + + + + + + + + + + Specifies the name of the exchange that the message was originally published + to. May be empty, meaning the default exchange. + + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + + This method delivers a message to the client, via a consumer. In the asynchronous + message delivery model, the client starts a consumer using the Consume method, then + the server responds with Deliver methods as and when messages arrive for that + consumer. + + + + + The server SHOULD track the number of times a message has been delivered to + clients and when a message is redelivered a certain number of times - e.g. 5 + times - without being acknowledged, the server SHOULD consider the message to be + unprocessable (possibly causing client applications to abort), and move the + message to a dead letter queue. + + + TODO. + + + + + + + + + + + + Specifies the name of the exchange that the message was originally published to. + May be empty, indicating the default exchange. + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + This method provides a direct access to the messages in a queue using a synchronous + dialogue that is designed for specific types of application where synchronous + functionality is more important than performance. + + + + + + + + + + + Specifies the name of the queue to get a message from. + + + + + + + This method delivers a message to the client following a get method. A message + delivered by 'get-ok' must be acknowledged unless the no-ack option was set in the + get method. + + + + + + + + + Specifies the name of the exchange that the message was originally published to. + If empty, the message was published to the default exchange. + + + + + Specifies the routing key name specified when the message was published. + + + + + + + + This method tells the client that the queue has no messages available for the + client. + + + + + + + + + + + This method acknowledges one or more messages delivered via the Deliver or Get-Ok + methods. The client can ask to confirm a single message or a set of messages up to + and including a specific message. + + + + + + + + If set to 1, the delivery tag is treated as "up to and including", so that the + client can acknowledge multiple messages with a single method. If set to zero, + the delivery tag refers to a single message. If the multiple field is 1, and the + delivery tag is zero, tells the server to acknowledge all outstanding messages. + + + + The server MUST validate that a non-zero delivery-tag refers to a delivered + message, and raise a channel exception if this is not the case. On a transacted + channel, this check MUST be done immediately and not delayed until a Tx.Commit. + Specifically, a client MUST not acknowledge the same message more than once. + + + TODO. + + + + + + + + + + This method allows a client to reject a message. It can be used to interrupt and + cancel large incoming messages, or return untreatable messages to their original + queue. + + + + + The server SHOULD be capable of accepting and process the Reject method while + sending message content with a Deliver or Get-Ok method. I.e. the server should + read and process incoming methods while sending output frames. To cancel a + partially-send content, the server sends a content body frame of size 1 (i.e. + with no data except the frame-end octet). + + + + + + The server SHOULD interpret this method as meaning that the client is unable to + process the message at this time. + + + TODO. + + + + + + The client MUST NOT use this method as a means of selecting messages to process. + + + TODO. + + + + + + + + + + If requeue is true, the server will attempt to requeue the message. If requeue + is false or the requeue attempt fails the messages are discarded or dead-lettered. + + + + + The server MUST NOT deliver the message to the same client within the + context of the current channel. The recommended strategy is to attempt to + deliver the message to an alternative consumer, and if that is not possible, + to move the message to a dead-letter queue. The server MAY use more + sophisticated tracking to hold the message on the queue and redeliver it to + the same client at a later stage. + + + TODO. + + + + + + + + + + This method asks the server to redeliver all unacknowledged messages on a + specified channel. Zero or more messages may be redelivered. This method + is deprecated in favour of the synchronous Recover/Recover-Ok. + + + + The server MUST set the redelivered flag on all messages that are resent. + + + TODO. + + + + + + If this field is zero, the message will be redelivered to the original + recipient. If this bit is 1, the server will attempt to requeue the message, + potentially then delivering it to an alternative subscriber. + + + + + + + + + This method asks the server to redeliver all unacknowledged messages on a + specified channel. Zero or more messages may be redelivered. This method + replaces the asynchronous Recover. + + + + The server MUST set the redelivered flag on all messages that are resent. + + + TODO. + + + + + + If this field is zero, the message will be redelivered to the original + recipient. If this bit is 1, the server will attempt to requeue the message, + potentially then delivering it to an alternative subscriber. + + + + + + + This method acknowledges a Basic.Recover method. + + + + + + + + + + The Tx class allows publish and ack operations to be batched into atomic + units of work. The intention is that all publish and ack requests issued + within a transaction will complete successfully or none of them will. + Servers SHOULD implement atomic transactions at least where all publish + or ack requests affect a single queue. Transactions that cover multiple + queues may be non-atomic, given that queues can be created and destroyed + asynchronously, and such events do not form part of any transaction. + Further, the behaviour of transactions with respect to the immediate and + mandatory flags on Basic.Publish methods is not defined. + + + + + Applications MUST NOT rely on the atomicity of transactions that + affect more than one queue. + + + + + Applications MUST NOT rely on the behaviour of transactions that + include messages published with the immediate option. + + + + + Applications MUST NOT rely on the behaviour of transactions that + include messages published with the mandatory option. + + + + + tx = C:SELECT S:SELECT-OK + / C:COMMIT S:COMMIT-OK + / C:ROLLBACK S:ROLLBACK-OK + + + + + + + + + + This method sets the channel to use standard transactions. The client must use this + method at least once on a channel before using the Commit or Rollback methods. + + + + + + + + This method confirms to the client that the channel was successfully set to use + standard transactions. + + + + + + + + + This method commits all message publications and acknowledgments performed in + the current transaction. A new transaction starts immediately after a commit. + + + + + + + The client MUST NOT use the Commit method on non-transacted channels. + + + The client opens a channel and then uses Tx.Commit. + + + + + + + This method confirms to the client that the commit succeeded. Note that if a commit + fails, the server raises a channel exception. + + + + + + + + + This method abandons all message publications and acknowledgments performed in + the current transaction. A new transaction starts immediately after a rollback. + Note that unacked messages will not be automatically redelivered by rollback; + if that is required an explicit recover call should be issued. + + + + + + + The client MUST NOT use the Rollback method on non-transacted channels. + + + The client opens a channel and then uses Tx.Rollback. + + + + + + + This method confirms to the client that the rollback succeeded. Note that if an + rollback fails, the server raises a channel exception. + + + + + + diff --git a/src/asynqp/amqptypes.py b/src/asynqp/amqptypes.py index dcead71..662f9ac 100644 --- a/src/asynqp/amqptypes.py +++ b/src/asynqp/amqptypes.py @@ -2,12 +2,6 @@ from . import serialisation -MAX_OCTET = 0xFF -MAX_SHORT = 0xFFFF -MAX_LONG = 0xFFFFFFFF -MAX_LONG_LONG = 0xFFFFFFFFFFFFFFFF - - class Bit(object): def __init__(self, value): if isinstance(value, type(self)): @@ -33,8 +27,11 @@ def read(cls, stream): class Octet(int): + MIN = 0 + MAX = (1 << 8) - 1 + def __new__(cls, value): - if not (0 <= value <= MAX_OCTET): + if not (Octet.MIN <= value <= Octet.MAX): raise TypeError('Could not construct an Octet from value {}'.format(value)) return super().__new__(cls, value) @@ -47,8 +44,11 @@ def read(cls, stream): class Short(int): + MIN = -(1 << 15) + MAX = (1 << 15) - 1 + def __new__(cls, value): - if not (0 <= value <= MAX_SHORT): + if not (Short.MIN <= value <= Short.MAX): raise TypeError('Could not construct a Short from value {}'.format(value)) return super().__new__(cls, value) @@ -60,9 +60,29 @@ def read(cls, stream): return cls(serialisation.read_short(stream)) +class UnsignedShort(int): + MIN = 0 + MAX = (1 << 16) - 1 + + def __new__(cls, value): + if not (UnsignedShort.MIN <= value <= UnsignedShort.MAX): + raise TypeError('Could not construct an UnsignedShort from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_short(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_short(stream)) + + class Long(int): + MIN = -(1 << 31) + MAX = (1 << 31) - 1 + def __new__(cls, value): - if not (0 <= value <= MAX_LONG): + if not (Long.MIN <= value <= Long.MAX): raise TypeError('Could not construct a Long from value {}'.format(value)) return super().__new__(cls, value) @@ -74,9 +94,29 @@ def read(cls, stream): return cls(serialisation.read_long(stream)) +class UnsignedLong(int): + MIN = 0 + MAX = (1 << 32) - 1 + + def __new__(cls, value): + if not (UnsignedLong.MIN <= value <= UnsignedLong.MAX): + raise TypeError('Could not construct a UnsignedLong from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_long(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_long(stream)) + + class LongLong(int): + MIN = -(1 << 63) + MAX = (1 << 63) - 1 + def __new__(cls, value): - if not (0 <= value <= MAX_LONG_LONG): + if not (LongLong.MIN <= value <= LongLong.MAX): raise TypeError('Could not construct a LongLong from value {}'.format(value)) return super().__new__(cls, value) @@ -88,9 +128,26 @@ def read(cls, stream): return cls(serialisation.read_long_long(stream)) +class UnsignedLongLong(int): + MIN = 0 + MAX = (1 << 64) - 1 + + def __new__(cls, value): + if not (UnsignedLongLong.MIN <= value <= UnsignedLongLong.MAX): + raise TypeError('Could not construct a UnsignedLongLong from value {}'.format(value)) + return super().__new__(cls, value) + + def write(self, stream): + stream.write(serialisation.pack_unsigned_long_long(self)) + + @classmethod + def read(cls, stream): + return cls(serialisation.read_unsigned_long_long(stream)) + + class ShortStr(str): def __new__(cls, value): - if len(value) > MAX_OCTET: + if len(value) > Octet.MAX: raise TypeError('Could not construct a ShortStr from value {}'.format(value)) return super().__new__(cls, value) @@ -107,7 +164,7 @@ def read(cls, stream): class LongStr(str): def __new__(cls, value): - if len(value) > MAX_LONG: + if len(value) > UnsignedLong.MAX: raise TypeError('Could not construct a LongStr from value {}'.format(value)) return super().__new__(cls, value) @@ -139,7 +196,7 @@ def __new__(cls, *args, **kwargs): raise TypeError("Could not construct a timestamp from value {}".format(value)) def __eq__(self, other): - return abs(self - other) < datetime.timedelta(seconds=1) + return abs(self - other) < datetime.timedelta(milliseconds=1) def write(self, stream): stamp = int(self.timestamp()) @@ -154,8 +211,11 @@ def read(cls, stream): 'bit': Bit, 'octet': Octet, 'short': Short, + 'unsignedshort': UnsignedShort, 'long': Long, + 'unsignedlong': UnsignedLong, 'longlong': LongLong, + 'unsignedlonglong': UnsignedLongLong, 'table': Table, 'longstr': LongStr, 'shortstr': ShortStr, diff --git a/src/asynqp/bases.py b/src/asynqp/bases.py deleted file mode 100644 index e9922c1..0000000 --- a/src/asynqp/bases.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -from . import spec -from . import frames -from .exceptions import AMQPError - - -class Sender(object): - def __init__(self, channel_id, protocol): - self.channel_id = channel_id - self.protocol = protocol - - def send_method(self, method): - self.protocol.send_method(self.channel_id, method) - - -class FrameHandler(object): - def __init__(self, synchroniser, sender): - self.synchroniser = synchroniser - self.sender = sender - - def handle(self, frame): - try: - meth = getattr(self, 'handle_' + type(frame).__name__) - except AttributeError: - meth = getattr(self, 'handle_' + type(frame.payload).__name__) - - meth(frame) diff --git a/src/asynqp/channel.py b/src/asynqp/channel.py index 500426d..09f6e33 100644 --- a/src/asynqp/channel.py +++ b/src/asynqp/channel.py @@ -1,16 +1,18 @@ import asyncio import re -from . import bases + from . import frames from . import spec from . import queue +from . import exceptions from . import exchange from . import message from . import routing -from .exceptions import UndeliverableMessage +from .exceptions import ( + UndeliverableMessage, AMQPError, ChannelClosed) +from .log import log -VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) VALID_EXCHANGE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])+$', flags=re.A) @@ -28,26 +30,45 @@ class Channel(object): the numerical ID of the channel """ - def __init__(self, id, synchroniser, sender, basic_return_consumer, queue_factory, reader): + def __init__(self, id, synchroniser, sender, basic_return_consumer, + queue_factory, reader, *, loop): + self._loop = loop self.id = id self.synchroniser = synchroniser self.sender = sender self.basic_return_consumer = basic_return_consumer self.queue_factory = queue_factory self.reader = reader + self._closed = False + # Indicates, that channel is closing by client(!) call + self._closing = False @asyncio.coroutine - def declare_exchange(self, name, type, *, durable=True, auto_delete=False, internal=False): + def declare_exchange(self, name, type, *, durable=True, auto_delete=False, + passive=False, internal=False, nowait=False, + arguments=None): """ Declare an :class:`Exchange` on the broker. If the exchange does not exist, it will be created. This method is a :ref:`coroutine `. :param str name: the name of the exchange. - :param str type: the type of the exchange (usually one of ``'fanout'``, ``'direct'``, ``'topic'``, or ``'headers'``) - :keyword bool durable: If true, the exchange will be re-created when the server restarts. - :keyword bool auto_delete: If true, the exchange will be deleted when the last queue is un-bound from it. - :keyword bool internal: If true, the exchange cannot be published to directly; it can only be bound to other exchanges. + :param str type: the type of the exchange + (usually one of ``'fanout'``, ``'direct'``, ``'topic'``, or ``'headers'``) + :keyword bool durable: If true, the exchange will be re-created when + the server restarts. + :keyword bool auto_delete: If true, the exchange will be + deleted when the last queue is un-bound from it. + :keyword bool passive: If `true` and exchange with such a name does + not exist it will raise a :class:`exceptions.NotFound`. If `false` + server will create it. Arguments ``durable``, ``auto_delete`` and + ``internal`` are ignored if `passive=True`. + :keyword bool internal: If true, the exchange cannot be published to + directly; it can only be bound to other exchanges. + :keyword bool nowait: If true, the method will not wait for declare-ok + to arrive and return right away. + :keyword dict arguments: Table of optional parameters for extensions to + the AMQP protocol. See :ref:`extensions`. :return: the new :class:`Exchange` object. """ @@ -55,18 +76,26 @@ def declare_exchange(self, name, type, *, durable=True, auto_delete=False, inter return exchange.Exchange(self.reader, self.synchroniser, self.sender, name, 'direct', True, False, False) if not VALID_EXCHANGE_NAME_RE.match(name): - raise ValueError("Invalid exchange name.\n" - "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " - "and do not begin with 'amq.'") - - self.sender.send_ExchangeDeclare(name, type, durable, auto_delete, internal) - yield from self.synchroniser.await(spec.ExchangeDeclareOK) - ex = exchange.Exchange(self.reader, self.synchroniser, self.sender, name, type, durable, auto_delete, internal) - self.reader.ready() + raise ValueError( + "Invalid exchange name.\n" + "Valid names consist of letters, digits, hyphen, underscore, " + "period, or colon, and do not begin with 'amq.'") + + self.sender.send_ExchangeDeclare( + name, type, passive, durable, auto_delete, internal, nowait, + arguments or {}) + if not nowait: + yield from self.synchroniser.wait(spec.ExchangeDeclareOK) + self.reader.ready() + ex = exchange.Exchange( + self.reader, self.synchroniser, self.sender, name, type, durable, + auto_delete, internal) return ex @asyncio.coroutine - def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=False): + def declare_queue(self, name='', *, durable=True, exclusive=False, + auto_delete=False, passive=False, + nowait=False, arguments=None): """ Declare a queue on the broker. If the queue does not exist, it will be created. @@ -74,31 +103,25 @@ def declare_queue(self, name='', *, durable=True, exclusive=False, auto_delete=F :param str name: the name of the queue. Supplying a name of '' will create a queue with a unique name of the server's choosing. - :keyword bool durable: If true, the queue will be re-created when the server restarts. - :keyword bool exclusive: If true, the queue can only be accessed by the current connection, and will be deleted when the connection is closed. - :keyword bool auto_delete: If true, the queue will be deleted when the last consumer is cancelled. If there were never any conusmers, the queue won't be deleted. + :keyword bool passive: If true and queue with such a name does not + exist it will raise a :class:`exceptions.NotFound` instead of + creating it. Arguments ``durable``, ``auto_delete`` and + ``exclusive`` are ignored if ``passive=True``. + :keyword bool nowait: If true, will not wait for a declare-ok to arrive. + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`Queue` object. """ - q = yield from self.queue_factory.declare(name, durable, exclusive, auto_delete) + q = yield from self.queue_factory.declare( + name, durable, exclusive, auto_delete, passive, nowait, + arguments if arguments is not None else {}) return q - @asyncio.coroutine - def close(self): - """ - Close the channel by handshaking with the server. - - This method is a :ref:`coroutine `. - """ - self.sender.send_Close(0, 'Channel closed by application', 0, 0) - yield from self.synchroniser.await(spec.ChannelCloseOK) - # don't call self.reader.ready - stop reading frames from the q - @asyncio.coroutine def set_qos(self, prefetch_size=0, prefetch_count=0, apply_globally=False): """ @@ -126,7 +149,7 @@ def set_qos(self, prefetch_size=0, prefetch_count=0, apply_globally=False): global=true to mean that the QoS settings should apply per-channel. """ self.sender.send_BasicQos(prefetch_size, prefetch_count, apply_globally) - yield from self.synchroniser.await(spec.BasicQosOK) + yield from self.synchroniser.wait(spec.BasicQosOK) self.reader.ready() def set_return_handler(self, handler): @@ -143,6 +166,33 @@ def set_return_handler(self, handler): """ self.basic_return_consumer.set_callback(handler) + def is_closed(self): + return self._closing or self._closed + + @asyncio.coroutine + def close(self): + """ + Close the channel by handshaking with the server. + + This method is a :ref:`coroutine `. + """ + # If we aren't already closed ask for server to close + if not self.is_closed(): + self._closing = True + # Let the ChannelActor do the actual close operations. + # It will do the work on CloseOK + self.sender.send_Close( + 0, 'Channel closed by application', 0, 0) + try: + yield from self.synchroniser.wait(spec.ChannelCloseOK) + except AMQPError: + # For example if both sides want to close or the connection + # is closed. + pass + else: + if self._closing: + log.warn("Called `close` on already closing channel...") + class ChannelFactory(object): def __init__(self, loop, protocol, dispatcher, connection_info): @@ -150,39 +200,77 @@ def __init__(self, loop, protocol, dispatcher, connection_info): self.protocol = protocol self.dispatcher = dispatcher self.connection_info = connection_info - self.next_channel_id = 1 + self.next_channel_id = 0 @asyncio.coroutine def open(self): - synchroniser = routing.Synchroniser() + self.next_channel_id += 1 + channel_id = self.next_channel_id + synchroniser = routing.Synchroniser(loop=self.loop) - sender = ChannelMethodSender(self.next_channel_id, self.protocol, self.connection_info) - basic_return_consumer = BasicReturnConsumer() + sender = ChannelMethodSender(channel_id, self.protocol, self.connection_info) + basic_return_consumer = BasicReturnConsumer(loop=self.loop) consumers = queue.Consumers(self.loop) consumers.add_consumer(basic_return_consumer) - handler = ChannelFrameHandler(synchroniser, sender) - reader, writer = routing.create_reader_and_writer(handler) - handler.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) + actor = ChannelActor(synchroniser, sender, loop=self.loop) + reader = routing.QueuedReader(actor, loop=self.loop) - queue_factory = queue.QueueFactory(sender, synchroniser, reader, consumers) - channel = Channel(self.next_channel_id, synchroniser, sender, basic_return_consumer, queue_factory, reader) + queue_factory = queue.QueueFactory( + sender, synchroniser, reader, consumers, loop=self.loop) + channel = Channel( + channel_id, synchroniser, sender, basic_return_consumer, + queue_factory, reader, loop=self.loop) - self.dispatcher.add_writer(self.next_channel_id, writer) + # Set actor dependencies + actor.message_receiver = MessageReceiver(synchroniser, sender, consumers, reader) + actor.consumers = consumers + actor.channel = channel + actor.reader = reader + + self.dispatcher.add_handler(channel_id, reader.feed) try: sender.send_ChannelOpen() reader.ready() - yield from synchroniser.await(spec.ChannelOpenOK) + yield from synchroniser.wait(spec.ChannelOpenOK) except: - self.dispatcher.remove_writer(self.next_channel_id) + # don't rollback self.next_channel_id; + # another call may have entered this method + # concurrently while we were yielding + # and we don't want to end up with duplicate channels. + # If we leave self.next_channel_id incremented, the worst + # that happens is we end up with non-sequential channel numbers. + # Small price to pay to keep this method re-entrant. + self.dispatcher.remove_handler(channel_id) raise - self.next_channel_id += 1 reader.ready() return channel -class ChannelFrameHandler(bases.FrameHandler): +class ChannelActor(routing.Actor): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Will set those in Channel factory + self.channel = None + self.consumers = None + self.message_receiver = None + self.reader = None + + def handle(self, frame): + # From docs on `close`: + # After sending this method, any received methods except Close and + # Close-OK MUST be discarded. + # So we will only process ChannelClose, ChannelCloseOK, PoisonPillFrame + # if channel is closed + if self.channel.is_closed(): + close_methods = (spec.ChannelClose, spec.ChannelCloseOK) + if isinstance(frame.payload, close_methods) or isinstance(frame, frames.PoisonPillFrame): + return super().handle(frame) + else: + return + return super().handle(frame) + def handle_ChannelOpenOK(self, frame): self.synchroniser.notify(spec.ChannelOpenOK) @@ -208,10 +296,8 @@ def handle_QueueDeleteOK(self, frame): self.synchroniser.notify(spec.QueueDeleteOK) def handle_BasicGetEmpty(self, frame): - self.synchroniser.notify(spec.BasicGetEmpty, False) - - def handle_BasicGetOK(self, frame): - asyncio.async(self.message_receiver.receive_getOK(frame)) + # Send result=None to notify Empty message + self.synchroniser.notify(spec.BasicGetEmpty, None) def handle_BasicConsumeOK(self, frame): self.synchroniser.notify(spec.BasicConsumeOK, frame.payload.consumer_tag) @@ -219,26 +305,72 @@ def handle_BasicConsumeOK(self, frame): def handle_BasicCancelOK(self, frame): self.synchroniser.notify(spec.BasicCancelOK) + def handle_BasicQosOK(self, frame): + self.synchroniser.notify(spec.BasicQosOK) + + def handle_BasicCancel(self, frame): + # Cancel consumer server-side + self.consumers.server_cancel(frame.payload.consumer_tag) + self.reader.ready() + + # Message receiving hanlers + + def handle_BasicGetOK(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + # Syncronizer will be notified after full msg is consumed + self.message_receiver.receive_getOK(frame) + def handle_BasicDeliver(self, frame): - asyncio.async(self.message_receiver.receive_deliver(frame)) + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_deliver(frame) def handle_ContentHeaderFrame(self, frame): - asyncio.async(self.message_receiver.receive_header(frame)) + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_header(frame) def handle_ContentBodyFrame(self, frame): - asyncio.async(self.message_receiver.receive_body(frame)) + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_body(frame) + + def handle_BasicReturn(self, frame): + assert self.message_receiver is not None, "message_receiver not set" + self.message_receiver.receive_return(frame) + + # Close handlers + + def handle_PoisonPillFrame(self, frame): + """ Is sent in case connection was closed or disconnected.""" + self._close_all(frame.exception) def handle_ChannelClose(self, frame): + """ AMQP server closed the channel with an error """ + # By docs: + # The response to receiving a Close after sending Close must be to + # send Close-Ok. + # + # No need for additional checks + self.sender.send_CloseOK() + exc = exceptions._get_exception_type(frame.payload.reply_code) + self._close_all(exc) def handle_ChannelCloseOK(self, frame): + """ AMQP server closed channel as per our request """ + assert self.channel._closing, "received a not expected CloseOk" + # Release the `close` method's future self.synchroniser.notify(spec.ChannelCloseOK) - def handle_BasicQosOK(self, frame): - self.synchroniser.notify(spec.BasicQosOK) + exc = ChannelClosed() + self._close_all(exc) - def handle_BasicReturn(self, frame): - asyncio.async(self.message_receiver.receive_return(frame)) + def _close_all(self, exc): + # Make sure all `close` calls don't deadlock + self.channel._closed = True + # If there were anyone who expected an `*-OK` kill them, as no data + # will follow after close. Any new calls should also raise an error. + self.synchroniser.killall(exc) + # Cancel all consumers with same error + self.consumers.error(exc) class MessageReceiver(object): @@ -248,10 +380,9 @@ def __init__(self, synchroniser, sender, consumers, reader): self.consumers = consumers self.reader = reader self.message_builder = None + self.is_getok_message = None - @asyncio.coroutine def receive_getOK(self, frame): - self.synchroniser.notify(spec.BasicGetOK, True) payload = frame.payload self.message_builder = message.MessageBuilder( self.sender, @@ -260,9 +391,10 @@ def receive_getOK(self, frame): payload.exchange, payload.routing_key ) + # Send message to synchroniser when done + self.is_getok_message = True self.reader.ready() - @asyncio.coroutine def receive_deliver(self, frame): payload = frame.payload self.message_builder = message.MessageBuilder( @@ -273,11 +405,11 @@ def receive_deliver(self, frame): payload.routing_key, payload.consumer_tag ) - self.reader.ready() - yield from self.async_receive() + # Delivers message to consumers when done + self.is_getok_message = False + self.reader.ready() - @asyncio.coroutine def receive_return(self, frame): payload = frame.payload self.message_builder = message.MessageBuilder( @@ -288,41 +420,44 @@ def receive_return(self, frame): payload.routing_key, BasicReturnConsumer.tag ) - self.reader.ready() - yield from self.async_receive() - - @asyncio.coroutine - def async_receive(self): - yield from self.synchroniser.await(frames.ContentHeaderFrame) - tag, msg = yield from self.synchroniser.await(frames.ContentBodyFrame) - - self.consumers.deliver(tag, msg) + # Delivers message to BasicReturnConsumer when done + self.is_getok_message = False self.reader.ready() - @asyncio.coroutine def receive_header(self, frame): - self.synchroniser.notify(frames.ContentHeaderFrame) + assert self.message_builder is not None, "Received unexpected header" self.message_builder.set_header(frame.payload) + if self.message_builder.done(): + self.message_done() + return self.reader.ready() - @asyncio.coroutine def receive_body(self, frame): + assert self.message_builder is not None, "Received unexpected body" self.message_builder.add_body_chunk(frame.payload) + if self.message_builder.done(): - msg = self.message_builder.build() - tag = self.message_builder.consumer_tag - self.synchroniser.notify(frames.ContentBodyFrame, (tag, msg)) - self.message_builder = None - # don't call self.reader.ready() if the message is all here - - # get() or async_receive() will call - # it when they have finished processing the completed msg + self.message_done() return + # If message is not done yet we still need more frames. Wait for them self.reader.ready() + def message_done(self): + msg = self.message_builder.build() + tag = self.message_builder.consumer_tag + if self.is_getok_message: + self.synchroniser.notify(spec.BasicGetOK, (tag, msg)) + # Dont call ready() if message arrive after GetOk. It's the + # ``Queue.get`` method's responsibility + else: + self.consumers.deliver(tag, msg) + self.reader.ready() + + self.message_builder = None + -# basically just a collection of aliases with some arguments hard coded for convenience -class ChannelMethodSender(bases.Sender): +class ChannelMethodSender(routing.Sender): def __init__(self, channel_id, protocol, connection_info): super().__init__(channel_id, protocol) self.connection_info = connection_info @@ -330,20 +465,20 @@ def __init__(self, channel_id, protocol, connection_info): def send_ChannelOpen(self): self.send_method(spec.ChannelOpen('')) - def send_ExchangeDeclare(self, name, type, durable, auto_delete, internal): - self.send_method(spec.ExchangeDeclare(0, name, type, False, durable, auto_delete, internal, False, {})) + def send_ExchangeDeclare(self, name, type, passive, durable, auto_delete, internal, nowait, arguments): + self.send_method(spec.ExchangeDeclare(0, name, type, passive, durable, auto_delete, internal, nowait, arguments)) def send_ExchangeDelete(self, name, if_unused): self.send_method(spec.ExchangeDelete(0, name, if_unused, False)) - def send_QueueDeclare(self, name, durable, exclusive, auto_delete): - self.send_method(spec.QueueDeclare(0, name, False, durable, exclusive, auto_delete, False, {})) + def send_QueueDeclare(self, name, durable, exclusive, auto_delete, passive, nowait, arguments): + self.send_method(spec.QueueDeclare(0, name, passive, durable, exclusive, auto_delete, nowait, arguments)) - def send_QueueBind(self, queue_name, exchange_name, routing_key): - self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, {})) + def send_QueueBind(self, queue_name, exchange_name, routing_key, arguments): + self.send_method(spec.QueueBind(0, queue_name, exchange_name, routing_key, False, arguments)) - def send_QueueUnbind(self, queue_name, exchange_name, routing_key): - self.send_method(spec.QueueUnbind(0, queue_name, exchange_name, routing_key, {})) + def send_QueueUnbind(self, queue_name, exchange_name, routing_key, arguments): + self.send_method(spec.QueueUnbind(0, queue_name, exchange_name, routing_key, arguments)) def send_QueuePurge(self, queue_name): self.send_method(spec.QueuePurge(0, queue_name, False)) @@ -355,8 +490,8 @@ def send_BasicPublish(self, exchange_name, routing_key, mandatory, message): self.send_method(spec.BasicPublish(0, exchange_name, routing_key, mandatory, False)) self.send_content(message) - def send_BasicConsume(self, queue_name, no_local, no_ack, exclusive): - self.send_method(spec.BasicConsume(0, queue_name, '', no_local, no_ack, exclusive, False, {})) + def send_BasicConsume(self, queue_name, no_local, no_ack, exclusive, arguments): + self.send_method(spec.BasicConsume(0, queue_name, '', no_local, no_ack, exclusive, False, arguments)) def send_BasicCancel(self, consumer_tag): self.send_method(spec.BasicCancel(consumer_tag, False)) @@ -384,7 +519,7 @@ def send_content(self, msg): header_frame = frames.ContentHeaderFrame(self.channel_id, header_payload) self.protocol.send_frame(header_frame) - for payload in message.get_frame_payloads(msg, self.connection_info.frame_max - 8): + for payload in message.get_frame_payloads(msg, self.connection_info['frame_max'] - 8): frame = frames.ContentBodyFrame(self.channel_id, payload) self.protocol.send_frame(frame) @@ -392,9 +527,9 @@ def send_content(self, msg): class BasicReturnConsumer(object): tag = -1 # a 'real' tag is a string so there will never be a clash - def __init__(self): + def __init__(self, *, loop): self.callback = self.default_behaviour - self.cancelled_future = asyncio.Future() + self.cancelled_future = asyncio.Future(loop=loop) def set_callback(self, callback): if callback is None: diff --git a/src/asynqp/connection.py b/src/asynqp/connection.py index 4e4dc31..401e000 100644 --- a/src/asynqp/connection.py +++ b/src/asynqp/connection.py @@ -1,16 +1,12 @@ import asyncio import sys -from . import channel -from . import bases from . import spec from . import routing - - -class ConnectionInfo(object): - def __init__(self, username, password, virtual_host): - self.username = username - self.password = password - self.virtual_host = virtual_host +from . import frames +from .channel import ChannelFactory +from .exceptions import ( + AMQPConnectionError, ConnectionClosed) +from .log import log class Connection(object): @@ -27,22 +23,28 @@ class Connection(object): Connections are created using :func:`asynqp.connect() `. - .. attribute:: closed + .. attribute:: transport + + The :class:`~asyncio.BaseTransport` over which the connection is communicating with the server + + .. attribute:: protocol - a :class:`~asyncio.Future` which is done when the handshake to close the connection has finished + The :class:`~asyncio.Protocol` which is paired with the transport """ - def __init__(self, loop, protocol, synchroniser, sender, dispatcher, connection_info): + def __init__(self, loop, transport, protocol, synchroniser, sender, dispatcher, connection_info): self.synchroniser = synchroniser self.sender = sender - self.channel_factory = channel.ChannelFactory(loop, protocol, dispatcher, connection_info) + self.channel_factory = ChannelFactory(loop, protocol, dispatcher, connection_info) self.connection_info = connection_info - # this is ugly. when the connection is closing, all methods other than ConnectionCloseOK - # should be ignored. at the moment this behaviour is part of the dispatcher - # but this introduces an extra dependency between Connection and ConnectionFrameHandler which - # i don't like - self.closing = asyncio.Future() - self.closing.add_done_callback(lambda fut: dispatcher.closing.set_result(fut.result())) + self.transport = transport + self.protocol = protocol + # Indicates, that close was initiated by client + self.closed = asyncio.Future(loop=loop) + # This future is a backport, so we don't need to log pending errors + self.closed.add_done_callback(lambda fut: fut.exception()) + + self._closing = False @asyncio.coroutine def open_channel(self): @@ -53,9 +55,18 @@ def open_channel(self): :return: The new :class:`Channel` object. """ + if self._closing: + raise ConnectionClosed("Closed by application") + if self.closed.done(): + raise self.closed.exception() + channel = yield from self.channel_factory.open() return channel + def is_closed(self): + " Returns True if connection was closed " + return self._closing or self.closed.done() + @asyncio.coroutine def close(self): """ @@ -63,60 +74,91 @@ def close(self): This method is a :ref:`coroutine `. """ - self.closing.set_result(True) - self.sender.send_Close(0, 'Connection closed by application', 0, 0) - yield from self.synchroniser.await(spec.ConnectionCloseOK) + if not self.is_closed(): + self._closing = True + # Let the ConnectionActor do the actual close operations. + # It will do the work on CloseOK + self.sender.send_Close( + 0, 'Connection closed by application', 0, 0) + try: + yield from self.synchroniser.wait(spec.ConnectionCloseOK) + except AMQPConnectionError: + # For example if both sides want to close or the connection + # is closed. + pass + else: + if self._closing: + log.warn("Called `close` on already closing connection...") + # finish all pending tasks + yield from self.protocol.heartbeat_monitor.wait_closed() @asyncio.coroutine -def open_connection(loop, protocol, dispatcher, connection_info): - synchroniser = routing.Synchroniser() +def open_connection(loop, transport, protocol, dispatcher, connection_info): + synchroniser = routing.Synchroniser(loop=loop) sender = ConnectionMethodSender(protocol) - connection = Connection(loop, protocol, synchroniser, sender, dispatcher, connection_info) - handler = ConnectionFrameHandler(synchroniser, sender, protocol, connection) - - reader, writer = routing.create_reader_and_writer(handler) + connection = Connection(loop, transport, protocol, synchroniser, sender, dispatcher, connection_info) + actor = ConnectionActor(synchroniser, sender, protocol, connection, dispatcher, loop=loop) + reader = routing.QueuedReader(actor, loop=loop) try: - dispatcher.add_writer(0, writer) + dispatcher.add_handler(0, reader.feed) protocol.send_protocol_header() reader.ready() - yield from synchroniser.await(spec.ConnectionStart) + yield from synchroniser.wait(spec.ConnectionStart) sender.send_StartOK( {"product": "asynqp", "version": "0.1", # todo: use pkg_resources to inspect the package - "platform": sys.version}, + "platform": sys.version, + "capabilities": { + "consumer_cancel_notify": True + }}, 'AMQPLAIN', - {'LOGIN': connection_info.username, 'PASSWORD': connection_info.password}, + {'LOGIN': connection_info['username'], 'PASSWORD': connection_info['password']}, 'en_US' ) reader.ready() - frame = yield from synchroniser.await(spec.ConnectionTune) + frame = yield from synchroniser.wait(spec.ConnectionTune) # just agree with whatever the server wants. Make this configurable in future - connection_info.frame_max = frame.payload.frame_max + connection_info['frame_max'] = frame.payload.frame_max heartbeat_interval = frame.payload.heartbeat sender.send_TuneOK(frame.payload.channel_max, frame.payload.frame_max, heartbeat_interval) - sender.send_Open(connection_info.virtual_host) + sender.send_Open(connection_info['virtual_host']) protocol.start_heartbeat(heartbeat_interval) reader.ready() - yield from synchroniser.await(spec.ConnectionOpenOK) + yield from synchroniser.wait(spec.ConnectionOpenOK) reader.ready() except: - dispatcher.remove_writer(0) + dispatcher.remove_handler(0) raise return connection -class ConnectionFrameHandler(bases.FrameHandler): - def __init__(self, synchroniser, sender, protocol, connection): - super().__init__(synchroniser, sender) +class ConnectionActor(routing.Actor): + def __init__(self, synchroniser, sender, protocol, connection, dispatcher, *, loop=None): + super().__init__(synchroniser, sender, loop=loop) self.protocol = protocol self.connection = connection + self.dispatcher = dispatcher + + def handle(self, frame): + # From docs on `close`: + # After sending this method, any received methods except Close and + # Close-OK MUST be discarded. + # So we will only process ConnectionClose, ConnectionCloseOK, + # PoisonPillFrame if channel is closed + if self.connection.is_closed(): + close_methods = (spec.ConnectionClose, spec.ConnectionCloseOK) + if isinstance(frame.payload, close_methods) or isinstance(frame, frames.PoisonPillFrame): + return super().handle(frame) + else: + return + return super().handle(frame) def handle_ConnectionStart(self, frame): self.synchroniser.notify(spec.ConnectionStart) @@ -127,17 +169,51 @@ def handle_ConnectionTune(self, frame): def handle_ConnectionOpenOK(self, frame): self.synchroniser.notify(spec.ConnectionOpenOK) + # Close handlers + + def handle_PoisonPillFrame(self, frame): + """ Is sent in case protocol lost connection to server.""" + # Will be delivered after Close or CloseOK handlers. It's for channels, + # so ignore it. + if self.connection.closed.done(): + return + # If connection was not closed already - we lost connection. + # Protocol should already be closed + self._close_all(frame.exception) + def handle_ConnectionClose(self, frame): - self.connection.closing.set_result(True) + """ AMQP server closed the channel with an error """ + # Notify server we are OK to close. self.sender.send_CloseOK() - self.protocol.transport.close() + + exc = ConnectionClosed(frame.payload.reply_text, + frame.payload.reply_code) + self._close_all(exc) + # This will not abort transport, it will try to flush remaining data + # asynchronously, as stated in `asyncio` docs. + self.protocol.close() def handle_ConnectionCloseOK(self, frame): - self.protocol.transport.close() self.synchroniser.notify(spec.ConnectionCloseOK) - - -class ConnectionMethodSender(bases.Sender): + exc = ConnectionClosed("Closed by application") + self._close_all(exc) + # We already agread with server on closing, so lets do it right away + self.protocol.close() + + def _close_all(self, exc): + # Make sure all `close` calls don't deadlock + self.connection.closed.set_exception(exc) + # Close heartbeat + self.protocol.heartbeat_monitor.stop() + # If there were anyone who expected an `*-OK` kill them, as no data + # will follow after close + self.synchroniser.killall(exc) + # Notify all channels about error + poison_frame = frames.PoisonPillFrame(exc) + self.dispatcher.dispatch_all(poison_frame) + + +class ConnectionMethodSender(routing.Sender): def __init__(self, protocol): super().__init__(0, protocol) diff --git a/src/asynqp/exceptions.py b/src/asynqp/exceptions.py index 7e942e1..4f94862 100644 --- a/src/asynqp/exceptions.py +++ b/src/asynqp/exceptions.py @@ -1,10 +1,61 @@ -class AMQPError(IOError): +from ._exceptions import AMQPError, AMQPChannelError +from .spec import EXCEPTIONS, CONSTANTS_INVERSE + + +__all__ = [ + "AMQPError", + "ConnectionLostError", + "ChannelClosed", + "ConnectionClosed", + "AMQPChannelError", + "AMQPConnectionError", + "UndeliverableMessage", + "Deleted" +] +__all__.extend(EXCEPTIONS.keys()) + + +class AMQPConnectionError(AMQPError): pass +class ConnectionLostError(AMQPConnectionError, ConnectionError): + """ Connection was closed unexpectedly """ + + def __init__(self, message, exc=None): + super().__init__(message) + self.original_exc = exc + + +class ConnectionClosed(AMQPConnectionError): + """ Connection was closed by client """ + + def __init__(self, reply_text, reply_code=None): + super().__init__(reply_text) + self.reply_text = reply_text + self.reply_code = reply_code + + +class ChannelClosed(AMQPChannelError): + """ Channel was closed by client """ + + class UndeliverableMessage(ValueError): pass class Deleted(ValueError): pass + + +class InvalidExchangeName(ValueError): + pass + + +globals().update(EXCEPTIONS) + + +def _get_exception_type(reply_code): + name = CONSTANTS_INVERSE[reply_code] + classname = ''.join([x.capitalize() for x in name.split('_')]) + return EXCEPTIONS[classname] diff --git a/src/asynqp/exchange.py b/src/asynqp/exchange.py index 78461b2..bcb7b61 100644 --- a/src/asynqp/exchange.py +++ b/src/asynqp/exchange.py @@ -38,6 +38,7 @@ def publish(self, message, routing_key, *, mandatory=True): :param asynqp.Message message: the message to send :param str routing_key: the routing key with which to publish the message + :param bool mandatory: if True (the default) undeliverable messages result in an error (see also :meth:`Channel.set_return_handler`) """ self.sender.send_BasicPublish(self.name, routing_key, mandatory, message) @@ -52,6 +53,5 @@ def delete(self, *, if_unused=True): it has no queues bound to it. """ self.sender.send_ExchangeDelete(self.name, if_unused) - yield from self.synchroniser.await(spec.ExchangeDeleteOK) + yield from self.synchroniser.wait(spec.ExchangeDeleteOK) self.reader.ready() - diff --git a/src/asynqp/frames.py b/src/asynqp/frames.py index 70d5055..784d6ea 100644 --- a/src/asynqp/frames.py +++ b/src/asynqp/frames.py @@ -64,3 +64,11 @@ class HeartbeatFrame(Frame): def __init__(self): pass + + +class PoisonPillFrame(Frame): + channel_id = 0 + payload = b'' + + def __init__(self, exception): + self.exception = exception diff --git a/src/asynqp/log.py b/src/asynqp/log.py new file mode 100644 index 0000000..c34e86a --- /dev/null +++ b/src/asynqp/log.py @@ -0,0 +1,3 @@ +import logging + +log = logging.getLogger("asynqp") diff --git a/src/asynqp/message.py b/src/asynqp/message.py index ce86ae2..9278c97 100644 --- a/src/asynqp/message.py +++ b/src/asynqp/message.py @@ -19,7 +19,9 @@ class Message(object): dicts will be converted to a string using JSON. :param dict headers: a dictionary of message headers :param str content_type: MIME content type - :param str content_encoding: MIME encoding + (defaults to 'application/json' if :code:`body` is a :class:`dict`, + or 'application/octet-stream' otherwise) + :param str content_encoding: MIME encoding (defaults to 'utf-8') :param int delivery_mode: 1 for non-persistent, 2 for persistent :param int priority: message priority - integer between 0 and 9 :param str correlation_id: correlation id of the message *(for applications)* @@ -75,23 +77,30 @@ def __init__(self, body, *, timestamp = timestamp if timestamp is not None else datetime.now() - self.properties = OrderedDict() + self._properties = OrderedDict() for name, amqptype in self.property_types.items(): value = locals()[name] if value is not None: value = amqptype(value) - self.properties[name] = value + self._properties[name] = value def __eq__(self, other): return (self.body == other.body - and self.properties == other.properties) + and self._properties == other._properties) def __getattr__(self, name): try: - return self.properties[name] + return self._properties[name] except KeyError as e: raise AttributeError from e + def __setattr__(self, name, value): + amqptype = self.property_types.get(name) + if amqptype is not None: + self._properties[name] = value if isinstance(value, amqptype) else amqptype(value) + return + super().__setattr__(name, value) + def json(self): """ Parse the message body as JSON. @@ -136,14 +145,14 @@ def reject(self, *, requeue=True): """ Reject the message. - :keyword bool redeliver: if true, the broker will attempt to requeue the + :keyword bool requeue: if true, the broker will attempt to requeue the message and deliver it to an alternate consumer. """ self.sender.send_BasicReject(self.delivery_tag, requeue) def get_header_payload(message, class_id): - return ContentHeaderPayload(class_id, len(message.body), list(message.properties.values())) + return ContentHeaderPayload(class_id, len(message.body), list(message._properties.values())) # NB: the total frame size will be 8 bytes larger than frame_body_size @@ -171,9 +180,9 @@ def __eq__(self, other): and self.properties == other.properties) def write(self, stream): - stream.write(serialisation.pack_short(self.class_id)) - stream.write(serialisation.pack_short(0)) # weight - stream.write(serialisation.pack_long_long(self.body_length)) + stream.write(serialisation.pack_unsigned_short(self.class_id)) + stream.write(serialisation.pack_unsigned_short(0)) # weight + stream.write(serialisation.pack_unsigned_long_long(self.body_length)) bytesio = BytesIO() @@ -186,27 +195,33 @@ def write(self, stream): val.write(bytesio) bitshift -= 1 - stream.write(serialisation.pack_short(property_flags)) + stream.write(serialisation.pack_unsigned_short(property_flags)) stream.write(bytesio.getvalue()) @classmethod def read(cls, raw): bytesio = BytesIO(raw) - class_id = serialisation.read_short(bytesio) - weight = serialisation.read_short(bytesio) + class_id = serialisation.read_unsigned_short(bytesio) + weight = serialisation.read_unsigned_short(bytesio) assert weight == 0 - body_length = serialisation.read_long_long(bytesio) - property_flags_short = serialisation.read_short(bytesio) + body_length = serialisation.read_unsigned_long_long(bytesio) + property_flags_short = serialisation.read_unsigned_short(bytesio) properties = [] - for amqptype, flag in zip(Message.property_types.values(), bin(property_flags_short)[2:]): - if flag == '1': + + for i, amqptype in enumerate(Message.property_types.values()): + pos = 15 - i # We started from `content_type` witch has pos==15 + if property_flags_short & (1 << pos): properties.append(amqptype.read(bytesio)) else: properties.append(None) return cls(class_id, body_length, properties) + def __repr__(self): + return "".format( + self.class_id, self.body_length, self.properties) + class MessageBuilder(object): def __init__(self, sender, delivery_tag, redelivered, exchange_name, routing_key, consumer_tag=None): diff --git a/src/asynqp/protocol.py b/src/asynqp/protocol.py index b7b1cb5..fbf8ee3 100644 --- a/src/asynqp/protocol.py +++ b/src/asynqp/protocol.py @@ -1,17 +1,20 @@ import asyncio import struct +from contextlib import suppress from . import spec from . import frames -from .routing import Dispatcher -from .exceptions import AMQPError +from .exceptions import AMQPError, ConnectionLostError +from .log import log class AMQP(asyncio.Protocol): - def __init__(self, dispatcher, loop): + def __init__(self, dispatcher, loop, close_callback=None): self.dispatcher = dispatcher self.partial_frame = b'' self.frame_reader = FrameReader() - self.heartbeat_monitor = HeartbeatMonitor(self, loop, 0) + self.heartbeat_monitor = HeartbeatMonitor(self, loop) + self._closed = False + self._close_callback = close_callback def connection_made(self, transport): self.transport = transport @@ -23,7 +26,7 @@ def data_received(self, data): try: result = self.frame_reader.read_frame(data) except AMQPError: - self.transport.close() + self.close() raise if result is None: # incomplete frame, wait for the rest @@ -31,8 +34,6 @@ def data_received(self, data): frame, remainder = result self.dispatcher.dispatch(frame) - if not remainder: - return data = remainder def send_method(self, channel, method): @@ -48,6 +49,35 @@ def send_protocol_header(self): def start_heartbeat(self, heartbeat_interval): self.heartbeat_monitor.start(heartbeat_interval) + def connection_lost(self, exc): + # If self._closed=True - we closed the transport ourselves. No need to + # dispatch PoisonPillFrame, as we should have closed everything already + if self._close_callback: + # _close_callback now only accepts coroutines + asyncio.ensure_future(self._close_callback(exc)) + + if not self._closed: + poison_exc = ConnectionLostError( + 'The connection was unexpectedly lost', exc) + self.dispatcher.dispatch_all(frames.PoisonPillFrame(poison_exc)) + # XXX: Really do we even need to raise this??? It's super bad API + raise poison_exc from exc + + def heartbeat_timeout(self): + """ Called by heartbeat_monitor on timeout """ + assert not self._closed, "Did we not stop heartbeat_monitor on close?" + log.error("Heartbeat time out") + poison_exc = ConnectionLostError('Heartbeat timed out') + poison_frame = frames.PoisonPillFrame(poison_exc) + self.dispatcher.dispatch_all(poison_frame) + # Spec says to just close socket without ConnectionClose handshake. + self.close() + + def close(self): + assert not self._closed, "Why do we close it 2-ce?" + self._closed = True + self.transport.close() + class FrameReader(object): def __init__(self): @@ -68,44 +98,71 @@ def read_frame(self, data): self.partial_frame = data return - raw_payload = data[7:7+size] - frame_end = data[7+size] + raw_payload = data[7:7 + size] + frame_end = data[7 + size] if frame_end != spec.FRAME_END: raise AMQPError("Frame end byte was incorrect") frame = frames.read(frame_type, channel_id, raw_payload) - remainder = data[8+size:] + remainder = data[8 + size:] return frame, remainder class HeartbeatMonitor(object): - def __init__(self, protocol, loop, heartbeat_interval): + def __init__(self, protocol, loop): self.protocol = protocol self.loop = loop - self.heartbeat_interval = heartbeat_interval - self.heartbeat_timeout_callback = None + self.send_hb_task = None + self.monitor_task = None + self._last_received = 0 def start(self, interval): - if interval > 0: - self.heartbeat_interval = interval - self.send_heartbeat() - self.monitor_heartbeat() - - def send_heartbeat(self): - if self.heartbeat_interval > 0: + if interval <= 0: + return + self.send_hb_task = asyncio.ensure_future(self.send_heartbeat(interval), loop=self.loop) + self.monitor_task = asyncio.ensure_future(self.monitor_heartbeat(interval), loop=self.loop) + + def stop(self): + if self.send_hb_task is not None: + self.send_hb_task.cancel() + if self.monitor_task is not None: + self.monitor_task.cancel() + + @asyncio.coroutine + def wait_closed(self): + if self.send_hb_task is not None: + with suppress(asyncio.CancelledError): + yield from self.send_hb_task + if self.monitor_task is not None: + with suppress(asyncio.CancelledError): + yield from self.monitor_task + + @asyncio.coroutine + def send_heartbeat(self, interval): + # XXX: Add `last_sent` frame monitoring to not send heartbeats + # if traffic was going through socket + while True: self.protocol.send_frame(frames.HeartbeatFrame()) - self.loop.call_later(self.heartbeat_interval, self.send_heartbeat) - - def monitor_heartbeat(self): - if self.heartbeat_interval > 0: - self.heartbeat_timeout_callback = self.loop.call_later(self.heartbeat_interval * 2, self.heartbeat_timed_out) + yield from asyncio.sleep(interval, loop=self.loop) + + @asyncio.coroutine + def monitor_heartbeat(self, interval): + self._last_received = self.loop.time() + no_beat_for = 0 + while True: + # As spec states: + # If a peer detects no incoming traffic (i.e. received octets) for + # two heartbeat intervals or longer, it should close the connection + yield from asyncio.sleep( + interval * 2 - no_beat_for, loop=self.loop) + + no_beat_for = self.loop.time() - self._last_received + if no_beat_for > interval * 2: + self.protocol.heartbeat_timeout() + self.send_hb_task.cancel() + return def heartbeat_received(self): - if self.heartbeat_timeout_callback is not None: - self.heartbeat_timeout_callback.cancel() - self.monitor_heartbeat() - - def heartbeat_timed_out(self): - self.protocol.send_method(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + self._last_received = self.loop.time() diff --git a/src/asynqp/queue.py b/src/asynqp/queue.py index fa76bc3..bbd07cf 100644 --- a/src/asynqp/queue.py +++ b/src/asynqp/queue.py @@ -1,8 +1,7 @@ import asyncio import re -from operator import delitem -from . import spec, frames -from .exceptions import Deleted +from . import spec +from .exceptions import Deleted, AMQPError, InvalidExchangeName VALID_QUEUE_NAME_RE = re.compile(r'^(?!amq\.)(\w|[-.:])*$', flags=re.A) @@ -32,8 +31,13 @@ class Queue(object): .. attribute:: auto_delete if True, the queue will be deleted when its last consumer is removed + + .. attribute:: arguments + + A dictionary of the extra arguments that were used to declare the queue. """ - def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete): + def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclusive, auto_delete, arguments, *, loop): + self._loop = loop self.reader = reader self.consumers = consumers self.synchroniser = synchroniser @@ -43,10 +47,11 @@ def __init__(self, reader, consumers, synchroniser, sender, name, durable, exclu self.durable = durable self.exclusive = exclusive self.auto_delete = auto_delete + self.arguments = arguments self.deleted = False @asyncio.coroutine - def bind(self, exchange, routing_key): + def bind(self, exchange, routing_key, *, arguments=None): """ Bind a queue to an exchange, with the supplied routing key. @@ -57,24 +62,35 @@ def bind(self, exchange, routing_key): :param asynqp.Exchange exchange: the :class:`Exchange` to bind to :param str routing_key: the routing key under which to bind + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The new :class:`QueueBinding` object """ if self.deleted: raise Deleted("Queue {} was deleted".format(self.name)) + if not exchange: + raise InvalidExchangeName("Can't bind queue {} to the default exchange".format(self.name)) - self.sender.send_QueueBind(self.name, exchange.name, routing_key) - yield from self.synchroniser.await(spec.QueueBindOK) + self.sender.send_QueueBind(self.name, exchange.name, routing_key, arguments or {}) + yield from self.synchroniser.wait(spec.QueueBindOK) b = QueueBinding(self.reader, self.sender, self.synchroniser, self, exchange, routing_key) self.reader.ready() return b @asyncio.coroutine - def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False): + def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False, arguments=None): """ Start a consumer on the queue. Messages will be delivered asynchronously to the consumer. The callback function will be called whenever a new message arrives on the queue. + Advanced usage: the callback object must be callable + (it must be a function or define a ``__call__`` method), + but may also define some further methods: + + * ``callback.on_cancel()``: called with no parameters when the consumer is successfully cancelled. + * ``callback.on_error(exc)``: called when the channel is closed due to an error. + The argument passed is the exception which caused the error. + This method is a :ref:`coroutine `. :param callable callback: a callback to be called when a message is delivered. @@ -83,15 +99,18 @@ def consume(self, callback, *, no_local=False, no_ack=False, exclusive=False): published by this connection. :keyword bool no_ack: If true, messages delivered to the consumer don't require acknowledgement. :keyword bool exclusive: If true, only this consumer can access the queue. + :keyword dict arguments: Table of optional parameters for extensions to the AMQP protocol. See :ref:`extensions`. :return: The newly created :class:`Consumer` object. """ if self.deleted: raise Deleted("Queue {} was deleted".format(self.name)) - self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive) - tag = yield from self.synchroniser.await(spec.BasicConsumeOK) - consumer = Consumer(tag, callback, self.sender, self.synchroniser, self.reader) + self.sender.send_BasicConsume(self.name, no_local, no_ack, exclusive, arguments or {}) + tag = yield from self.synchroniser.wait(spec.BasicConsumeOK) + consumer = Consumer( + tag, callback, self.sender, self.synchroniser, self.reader, + loop=self._loop) self.consumers.add_consumer(consumer) self.reader.ready() return consumer @@ -112,11 +131,10 @@ def get(self, *, no_ack=False): raise Deleted("Queue {} was deleted".format(self.name)) self.sender.send_BasicGet(self.name, no_ack) - has_message = yield from self.synchroniser.await(spec.BasicGetOK, spec.BasicGetEmpty) + tag_msg = yield from self.synchroniser.wait(spec.BasicGetOK, spec.BasicGetEmpty) - if has_message: - yield from self.synchroniser.await(frames.ContentHeaderFrame) - consumer_tag, msg = yield from self.synchroniser.await(frames.ContentBodyFrame) + if tag_msg is not None: + consumer_tag, msg = tag_msg assert consumer_tag is None else: msg = None @@ -131,7 +149,7 @@ def purge(self): This method is a :ref:`coroutine `. """ self.sender.send_QueuePurge(self.name) - yield from self.synchroniser.await(spec.QueuePurgeOK) + yield from self.synchroniser.wait(spec.QueuePurgeOK) self.reader.ready() @asyncio.coroutine @@ -150,7 +168,7 @@ def delete(self, *, if_unused=True, if_empty=True): raise Deleted("Queue {} was already deleted".format(self.name)) self.sender.send_QueueDelete(self.name, if_unused, if_empty) - yield from self.synchroniser.await(spec.QueueDeleteOK) + yield from self.synchroniser.wait(spec.QueueDeleteOK) self.deleted = True self.reader.ready() @@ -188,7 +206,7 @@ def __init__(self, reader, sender, synchroniser, queue, exchange, routing_key): self.deleted = False @asyncio.coroutine - def unbind(self): + def unbind(self, arguments=None): """ Unbind the queue from the exchange. @@ -197,8 +215,8 @@ def unbind(self): if self.deleted: raise Deleted("Queue {} was already unbound from exchange {}".format(self.queue.name, self.exchange.name)) - self.sender.send_QueueUnbind(self.queue.name, self.exchange.name, self.routing_key) - yield from self.synchroniser.await(spec.QueueUnbindOK) + self.sender.send_QueueUnbind(self.queue.name, self.exchange.name, self.routing_key, arguments or {}) + yield from self.synchroniser.wait(spec.QueueUnbindOK) self.deleted = True self.reader.ready() @@ -223,14 +241,15 @@ class Consumer(object): Boolean. True if the consumer has been successfully cancelled. """ - def __init__(self, tag, callback, sender, synchroniser, reader): + def __init__(self, tag, callback, sender, synchroniser, reader, *, loop): + self._loop = loop self.tag = tag self.callback = callback self.sender = sender self.cancelled = False self.synchroniser = synchroniser self.reader = reader - self.cancelled_future = asyncio.Future() + self.cancelled_future = asyncio.Future(loop=self._loop) @asyncio.coroutine def cancel(self): @@ -240,30 +259,47 @@ def cancel(self): This method is a :ref:`coroutine `. """ self.sender.send_BasicCancel(self.tag) - yield from self.synchroniser.await(spec.BasicCancelOK) + try: + yield from self.synchroniser.wait(spec.BasicCancelOK) + except AMQPError: + pass + else: + # No need to call ready if channel closed. + self.reader.ready() self.cancelled = True self.cancelled_future.set_result(self) - self.reader.ready() + if hasattr(self.callback, 'on_cancel'): + self.callback.on_cancel() class QueueFactory(object): - def __init__(self, sender, synchroniser, reader, consumers): + def __init__(self, sender, synchroniser, reader, consumers, *, loop): + self._loop = loop self.sender = sender self.synchroniser = synchroniser self.reader = reader self.consumers = consumers @asyncio.coroutine - def declare(self, name, durable, exclusive, auto_delete): - if not VALID_QUEUE_NAME_RE.match(name): - raise ValueError("Not a valid queue name.\n" - "Valid names consist of letters, digits, hyphen, underscore, period, or colon, " - "and do not begin with 'amq.'") - - self.sender.send_QueueDeclare(name, durable, exclusive, auto_delete) - name = yield from self.synchroniser.await(spec.QueueDeclareOK) - q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, name, durable, exclusive, auto_delete) - self.reader.ready() + def declare(self, name, durable, exclusive, auto_delete, passive, nowait, + arguments): + if not (name == 'amq.rabbitmq.reply-to' or VALID_QUEUE_NAME_RE.match(name)): + raise ValueError( + "Not a valid queue name.\n" + "Valid names consist of letters, digits, hyphen, underscore, " + "period, or colon, and do not begin with 'amq.'") + if not name and nowait: + raise ValueError("Declaring the queue without `name` and with " + "`nowait` is forbidden") + + self.sender.send_QueueDeclare( + name, durable, exclusive, auto_delete, passive, nowait, arguments) + if not nowait: + name = yield from self.synchroniser.wait(spec.QueueDeclareOK) + self.reader.ready() + q = Queue(self.reader, self.consumers, self.synchroniser, self.sender, + name, durable, exclusive, auto_delete, arguments, + loop=self._loop) return q @@ -275,9 +311,20 @@ def __init__(self, loop): def add_consumer(self, consumer): self.consumers[consumer.tag] = consumer # so the consumer gets garbage collected when it is cancelled - consumer.cancelled_future.add_done_callback(lambda fut: delitem(self.consumers, fut.result().tag)) + consumer.cancelled_future.add_done_callback(lambda fut: self.consumers.pop(fut.result().tag, None)) def deliver(self, tag, msg): assert tag in self.consumers, "Message got delivered to a non existent consumer" consumer = self.consumers[tag] self.loop.call_soon(consumer.callback, msg) + + def server_cancel(self, tag): + consumer = self.consumers.pop(tag, None) + if consumer: + if hasattr(consumer.callback, 'on_cancel'): + consumer.callback.on_cancel() + + def error(self, exc): + for consumer in self.consumers.values(): + if hasattr(consumer.callback, 'on_error'): + consumer.callback.on_error(exc) diff --git a/src/asynqp/routing.py b/src/asynqp/routing.py index 59280eb..fb5cda2 100644 --- a/src/asynqp/routing.py +++ b/src/asynqp/routing.py @@ -1,126 +1,126 @@ -import asyncio -import collections -from . import frames -from . import spec - - -_TEST = False - - -class Dispatcher(object): - def __init__(self): - self.queue_writers = {} - self.closing = asyncio.Future() - - def add_writer(self, channel_id, writer): - self.queue_writers[channel_id] = writer - - def remove_writer(self, channel_id): - del self.queue_writers[channel_id] - - def dispatch(self, frame): - if isinstance(frame, frames.HeartbeatFrame): - return - if self.closing.done() and not isinstance(frame.payload, (spec.ConnectionClose, spec.ConnectionCloseOK)): - return - writer = self.queue_writers[frame.channel_id] - writer.enqueue(frame) - - -class Synchroniser(object): - def __init__(self): - self._futures = OrderedManyToManyMap() - - def await(self, *expected_methods): - fut = asyncio.Future() - self._futures.add_item(expected_methods, fut) - return fut - - def notify(self, method, result=None): - fut = self._futures.get_leftmost(method) - fut.set_result(result) - self._futures.remove_item(fut) - - -def create_reader_and_writer(handler): - q = asyncio.Queue() - reader = QueueReader(handler, q) - writer = QueueWriter(q) - return reader, writer - - -# When ready() is called, wait for a frame to arrive on the queue. -# When the frame does arrive, dispatch it to the handler and do nothing -# until someone calls ready() again. -class QueueReader(object): - def __init__(self, handler, q): - self.handler = handler - self.q = q - self.is_waiting = False - - def ready(self): - assert not self.is_waiting, "ready() got called while waiting for a frame to be read" - self.is_waiting = True - t = asyncio.async(self._read_next()) - if _TEST: # this feels hacky to me - t._log_destroy_pending = False - - @asyncio.coroutine - def _read_next(self): - assert self.is_waiting, "a frame got read without ready() having been called" - frame = yield from self.q.get() - self.is_waiting = False - self.handler.handle(frame) - - -class QueueWriter(object): - def __init__(self, q): - self.q = q - - def enqueue(self, frame): - self.q.put_nowait(frame) - - -class OrderedManyToManyMap(object): - def __init__(self): - self._items = collections.defaultdict(OrderedSet) - - def add_item(self, keys, item): - for key in keys: - self._items[key].add(item) - - def remove_item(self, item): - for ordered_set in self._items.values(): - ordered_set.discard(item) - - def get_leftmost(self, key): - return self._items[key].first() - - -class OrderedSet(collections.MutableSet): - def __init__(self): - self._map = collections.OrderedDict() - - def __contains__(self, item): - return item in self._map - - def __iter__(self): - return iter(self._map.keys()) - - def __getitem__(self, ix): - return - - def __len__(self): - return len(self._map) - - def add(self, item): - self._map[item] = None - - def discard(self, item): - try: - del self._map[item] - except KeyError: - pass - - def first(self): - return next(iter(self._map)) +import asyncio +import collections +from . import frames +from .log import log + + +class Dispatcher(object): + def __init__(self): + self.handlers = {} + + def add_handler(self, channel_id, handler): + self.handlers[channel_id] = handler + + def remove_handler(self, channel_id): + del self.handlers[channel_id] + + def dispatch(self, frame): + if isinstance(frame, frames.HeartbeatFrame): + return + handler = self.handlers[frame.channel_id] + handler(frame) + + def dispatch_all(self, frame): + for handler in self.handlers.values(): + handler(frame) + + +class Sender(object): + def __init__(self, channel_id, protocol): + self.channel_id = channel_id + self.protocol = protocol + + def send_method(self, method): + self.protocol.send_method(self.channel_id, method) + + +class Actor(object): + def __init__(self, synchroniser, sender, *, loop): + self._loop = loop + self.synchroniser = synchroniser + self.sender = sender + + def handle(self, frame): + try: + meth = getattr(self, 'handle_' + type(frame).__name__) + except AttributeError: + meth = getattr(self, 'handle_' + type(frame.payload).__name__) + + meth(frame) + + +class Synchroniser(object): + + def __init__(self, *, loop): + self._loop = loop + self._futures = collections.defaultdict(collections.deque) + self.connection_exc = None + + def wait(self, *expected_methods): + fut = asyncio.Future(loop=self._loop) + + if self.connection_exc is not None: + fut.set_exception(self.connection_exc) + return fut + + for method in expected_methods: + self._futures[method].append(fut) + return fut + + def notify(self, method, result=None): + while True: + try: + fut = self._futures[method].popleft() + except IndexError: + # XXX: we can't just ignore this. + log.error("Got an unexpected method notification %s", method) + return + # We can have done futures if they were awaited together, like + # (spec.BasicGetOK, spec.BasicGetEmpty). + if not fut.done(): + break + + fut.set_result(result) + + def killall(self, exc): + """ Connection/Channel was closed. All subsequent and ongoing requests + should raise an error + """ + self.connection_exc = exc + # Set an exception for all others + for method, futs in self._futures.items(): + for fut in futs: + if fut.done(): + continue + fut.set_exception(exc) + self._futures.clear() + + +# When ready() is called, wait for a frame to arrive on the queue. +# When the frame does arrive, dispatch it to the handler and do nothing +# until someone calls ready() again. +class QueuedReader(object): + def __init__(self, handler, *, loop): + self.handler = handler + self.is_waiting = False + self.pending_frames = collections.deque() + self._loop = loop + + def ready(self): + assert not self.is_waiting, "ready() got called while waiting for a frame to be read" + if self.pending_frames: + frame = self.pending_frames.popleft() + # We will call it in another tick just to be more strict about the + # sequence of frames + self._loop.call_soon(self.handler.handle, frame) + else: + self.is_waiting = True + + def feed(self, frame): + if self.is_waiting: + self.is_waiting = False + # We will call it in another tick just to be more strict about the + # sequence of frames + self._loop.call_soon(self.handler.handle, frame) + else: + self.pending_frames.append(frame) diff --git a/src/asynqp/serialisation.py b/src/asynqp/serialisation.py index d3add45..694a369 100644 --- a/src/asynqp/serialisation.py +++ b/src/asynqp/serialisation.py @@ -1,5 +1,6 @@ import struct from .exceptions import AMQPError +from datetime import datetime, timezone def rethrow_as(expected_cls, to_throw): @@ -28,16 +29,31 @@ def read_short(stream): return _read_short(stream)[0] +@rethrow_as(struct.error, AMQPError('failed to read an unsigned short')) +def read_unsigned_short(stream): + return _read_unsigned_short(stream)[0] + + @rethrow_as(struct.error, AMQPError('failed to read a long')) def read_long(stream): return _read_long(stream)[0] -@rethrow_as(struct.error, AMQPError('failed to read a long')) +@rethrow_as(struct.error, AMQPError('failed to read an unsigned long')) +def read_unsigned_long(stream): + return _read_unsigned_long(stream)[0] + + +@rethrow_as(struct.error, AMQPError('failed to read a long long')) def read_long_long(stream): return _read_long_long(stream)[0] +@rethrow_as(struct.error, AMQPError('failed to read an unsigned long long')) +def read_unsigned_long_long(stream): + return _read_unsigned_long_long(stream)[0] + + @rethrow_as(struct.error, AMQPError('failed to read a short string')) def read_short_string(stream): return _read_short_string(stream)[0] @@ -54,6 +70,12 @@ def read_table(stream): return _read_table(stream)[0] +@rethrow_as(KeyError, AMQPError('failed to read an array')) +@rethrow_as(struct.error, AMQPError('failed to read an array')) +def read_array(stream): + return _read_array(stream)[0] + + @rethrow_as(struct.error, AMQPError('failed to read a boolean')) def read_bool(stream): return _read_bool(stream)[0] @@ -66,20 +88,37 @@ def read_bools(byte, number_of_bools): return (b == "1" for b in reversed(bits)) -def _read_table(stream): - # TODO: more value types +@rethrow_as(struct.error, AMQPError('failed to read a boolean')) +def read_timestamp(stream): + return _read_timestamp(stream)[0] + + +def qpid_rabbit_mq_table(): + # TODO: fix amqp 0.9.1 compatibility + # TODO: Add missing types TABLE_VALUE_PARSERS = { b't': _read_bool, - b's': _read_short_string, + b'b': _read_signed_byte, + b's': _read_short, + b'I': _read_long, + b'l': _read_long_long, + b'f': _read_float, b'S': _read_long_string, - b'F': _read_table + b'A': _read_array, + b'V': _read_void, + b'x': _read_byte_array, + b'F': _read_table, + b'T': _read_timestamp } + return TABLE_VALUE_PARSERS + - consumed = 0 +def _read_table(stream): + TABLE_VALUE_PARSERS = qpid_rabbit_mq_table() table = {} - table_length, initial_long_size = _read_long(stream) - consumed += initial_long_size + table_length, initial_long_size = _read_unsigned_long(stream) + consumed = initial_long_size while consumed < table_length + initial_long_size: key, x = _read_short_string(stream) @@ -103,11 +142,11 @@ def _read_short_string(stream): def _read_long_string(stream): - str_length, x = _read_long(stream) - bytestring = stream.read(str_length) - if len(bytestring) != str_length: + str_length, x = _read_unsigned_long(stream) + buffer = stream.read(str_length) + if len(buffer) != str_length: raise AMQPError("Long string had incorrect length") - return bytestring.decode('utf-8'), x + str_length + return buffer.decode('utf-8'), x + str_length def _read_octet(stream): @@ -115,68 +154,196 @@ def _read_octet(stream): return x, 1 +def _read_signed_byte(stream): + x, = struct.unpack_from('!b', stream.read(1)) + return x, 1 + + def _read_bool(stream): x, = struct.unpack('!?', stream.read(1)) return x, 1 def _read_short(stream): + x, = struct.unpack('!h', stream.read(2)) + return x, 2 + + +def _read_unsigned_short(stream): x, = struct.unpack('!H', stream.read(2)) return x, 2 def _read_long(stream): + x, = struct.unpack('!l', stream.read(4)) + return x, 4 + + +def _read_unsigned_long(stream): x, = struct.unpack('!L', stream.read(4)) return x, 4 def _read_long_long(stream): + x, = struct.unpack('!q', stream.read(8)) + return x, 8 + + +def _read_unsigned_long_long(stream): x, = struct.unpack('!Q', stream.read(8)) return x, 8 +def _read_float(stream): + x, = struct.unpack('!f', stream.read(4)) + return x, 4 + + +def _read_timestamp(stream): + x, = struct.unpack('!Q', stream.read(8)) + # From datetime.fromutctimestamp converts it to a local timestamp without timezone information + return datetime.fromtimestamp(x * 1e-3, timezone.utc), 8 + + +def _read_array(stream): + TABLE_VALUE_PARSERS = qpid_rabbit_mq_table() + field_array = [] + + # The standard says only long, but unsigned long seems sensible + array_length, initial_long_size = _read_unsigned_long(stream) + consumed = initial_long_size + + while consumed < array_length + initial_long_size: + value_type_code = stream.read(1) + consumed += 1 + value, x = TABLE_VALUE_PARSERS[value_type_code](stream) + consumed += x + field_array.append(value) + + return field_array, consumed + + +def _read_void(stream): + return None, 0 + + +def _read_byte_array(stream): + byte_array_length, x = _read_unsigned_long(stream) + return stream.read(byte_array_length), byte_array_length + x + + ########################################################### # Serialisation ########################################################### def pack_short_string(string): - bytes = string.encode('utf-8') - return pack_octet(len(bytes)) + bytes + buffer = string.encode('utf-8') + return pack_octet(len(buffer)) + buffer def pack_long_string(string): - bytes = string.encode('utf-8') - return pack_long(len(bytes)) + bytes + buffer = string.encode('utf-8') + return pack_unsigned_long(len(buffer)) + buffer + + +def pack_field_value(value): + if value is None: + return b'V' + if isinstance(value, bool): + return b't' + pack_bool(value) + if isinstance(value, dict): + return b'F' + pack_table(value) + if isinstance(value, list): + return b'A' + pack_array(value) + if isinstance(value, bytes): + return b'x' + pack_byte_array(value) + if isinstance(value, str): + return b'S' + pack_long_string(value) + if isinstance(value, datetime): + return b'T' + pack_timestamp(value) + if isinstance(value, int): + if value.bit_length() < 8: + return b'b' + pack_signed_byte(value) + if value.bit_length() < 32: + return b'I' + pack_long(value) + if isinstance(value, float): + return b'f' + pack_float(value) + raise NotImplementedError() def pack_table(d): - bytes = b'' + buffer = b'' for key, value in d.items(): - if not isinstance(value, str): - raise NotImplementedError() - bytes += pack_short_string(key) - bytes += b'S' # todo: more values - bytes += pack_long_string(value) - val = pack_long(len(bytes)) + bytes - return val + buffer += pack_short_string(key) + # todo: more values + buffer += pack_field_value(value) + + return pack_unsigned_long(len(buffer)) + buffer def pack_octet(number): return struct.pack('!B', number) +def pack_signed_byte(number): + return struct.pack('!b', number) + + +def pack_unsigned_byte(number): + return struct.pack('!B', number) + + def pack_short(number): + return struct.pack('!h', number) + + +def pack_unsigned_short(number): return struct.pack('!H', number) def pack_long(number): + return struct.pack('!l', number) + + +def pack_unsigned_long(number): return struct.pack('!L', number) def pack_long_long(number): + return struct.pack('!q', number) + + +def pack_unsigned_long_long(number): return struct.pack('!Q', number) +def pack_float(number): + return struct.pack('!f', number) + + +def pack_bool(b): + return struct.pack('!?', b) + + +def pack_timestamp(timeval): + number = int(timeval.timestamp() * 1e3) + return struct.pack('!Q', number) + + +def pack_byte_array(value): + buffer = pack_unsigned_long(len(value)) + buffer += value + return buffer + + +def pack_array(items): + buffer = b'' + for value in items: + buffer += pack_field_value(value) + + return pack_unsigned_long(len(buffer)) + buffer + + def pack_bools(*bs): tot = 0 for n, b in enumerate(bs): diff --git a/src/asynqp/spec.py b/src/asynqp/spec.py index 2c7d9ac..72a7aad 100644 --- a/src/asynqp/spec.py +++ b/src/asynqp/spec.py @@ -6,6 +6,7 @@ from . import amqptypes from . import serialisation from .amqptypes import FIELD_TYPES +from ._exceptions import AMQPError def read_method(raw): @@ -164,7 +165,20 @@ def generate_methods(classes): return methods +def generate_exceptions(constants): + ret = {} + + for name, value in constants.items(): + if 300 <= value < 600: # it's an error + classname = ''.join([x.capitalize() for x in name.split('_')]) + ret[classname] = type(classname, (AMQPError,), {}) + + return ret + + METHODS, CONSTANTS = load_spec() +CONSTANTS_INVERSE = {value: name for name, value in CONSTANTS.items()} +EXCEPTIONS = generate_exceptions(CONSTANTS) # what the hack? 'response' is always a table but the protocol spec says it's a longstr. METHODS['ConnectionStartOK'].field_info['response'] = amqptypes.Table diff --git a/test/__init__.py b/test/__init__.py index e69de29..96ddb0e 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -0,0 +1,10 @@ +import asyncio +import logging +from .util import testing_exception_handler + + +loop = asyncio.get_event_loop() +loop.set_exception_handler(testing_exception_handler) + + +logging.getLogger('asynqp').setLevel(100) # mute the logger diff --git a/test/base_contexts.py b/test/base_contexts.py index 12709ba..7603776 100644 --- a/test/base_contexts.py +++ b/test/base_contexts.py @@ -1,39 +1,45 @@ import asyncio import asynqp import asynqp.routing -from asyncio import test_utils from asynqp import spec from asynqp import protocol -from asynqp.connection import ConnectionInfo, open_connection +from asynqp.connection import open_connection from unittest import mock -from .util import MockServer, FakeTransport +from .util import MockServer, FakeTransport, run_briefly class LoopContext: def given_an_event_loop(self): + self.exceptions = [] self.loop = asyncio.get_event_loop() - asynqp.routing._TEST = True - - def tick(self): - test_utils.run_briefly(self.loop) + self.loop.set_debug(True) def cleanup_test_hack(self): - asynqp.routing._TEST = False + self.loop.set_debug(False) + + def exception_handler(self, loop, context): + self.exceptions.append(context['exception']) + + def tick(self): + run_briefly(self.loop) def async_partial(self, coro): """ Schedule a coroutine which you are not expecting to complete before the end of the test. Disables the error log when the task is destroyed before completing. """ - t = asyncio.async(coro) + t = asyncio.ensure_future(coro) t._log_destroy_pending = False self.tick() return t + def wait_for(self, coro): + return self.loop.run_until_complete(asyncio.wait_for(coro, timeout=0.2, loop=self.loop)) + class MockServerContext(LoopContext): def given_a_mock_server_on_the_other_end_of_the_transport(self): - self.dispatcher = protocol.Dispatcher() + self.dispatcher = asynqp.routing.Dispatcher() self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.server = MockServer(self.protocol, self.tick) self.transport = FakeTransport(self.server) @@ -42,8 +48,8 @@ def given_a_mock_server_on_the_other_end_of_the_transport(self): class OpenConnectionContext(MockServerContext): def given_an_open_connection(self): - connection_info = ConnectionInfo('guest', 'guest', '/') - task = asyncio.async(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} + task = asyncio.ensure_future(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.tick() start_method = spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US') @@ -57,22 +63,27 @@ def given_an_open_connection(self): self.connection = task.result() + def cleanup_connection(self): + self.connection.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + self.connection.protocol.heartbeat_monitor.wait_closed()) + class OpenChannelContext(OpenConnectionContext): def given_an_open_channel(self): self.channel = self.open_channel() def open_channel(self, channel_id=1): - task = asyncio.async(self.connection.open_channel(), loop=self.loop) + task = asyncio.ensure_future(self.connection.open_channel(), loop=self.loop) self.tick() self.server.send_method(channel_id, spec.ChannelOpenOK('')) - return task.result() + return self.loop.run_until_complete(task) class QueueContext(OpenChannelContext): def given_a_queue(self): queue_name = 'my.nice.queue' - task = asyncio.async(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), loop=self.loop) + task = asyncio.ensure_future(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.QueueDeclareOK(queue_name, 123, 456)) self.queue = task.result() @@ -83,8 +94,8 @@ def given_an_exchange(self): self.exchange = self.make_exchange('my.nice.exchange') def make_exchange(self, name): - task = asyncio.async(self.channel.declare_exchange(name, 'fanout', durable=True, auto_delete=False, internal=False), - loop=self.loop) + task = asyncio.ensure_future(self.channel.declare_exchange(name, 'fanout', durable=True, auto_delete=False, internal=False), + loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.ExchangeDeclareOK()) return task.result() @@ -92,7 +103,7 @@ def make_exchange(self, name): class BoundQueueContext(QueueContext, ExchangeContext): def given_a_bound_queue(self): - task = asyncio.async(self.queue.bind(self.exchange, 'routing.key')) + task = asyncio.ensure_future(self.queue.bind(self.exchange, 'routing.key')) self.tick() self.server.send_method(self.channel.id, spec.QueueBindOK()) self.binding = task.result() @@ -103,53 +114,16 @@ def given_a_consumer(self): self.callback = mock.Mock() del self.callback._is_coroutine # :( - task = asyncio.async(self.queue.consume(self.callback, no_local=False, no_ack=False, exclusive=False)) + task = asyncio.ensure_future(self.queue.consume(self.callback, no_local=False, no_ack=False, exclusive=False)) self.tick() self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) self.consumer = task.result() -class MockLoopContext(LoopContext): - def given_an_event_loop(self): - self.loop = mock.Mock(spec=asyncio.AbstractEventLoop) - - -class LegacyConnectionContext(LoopContext): - def given_the_pieces_i_need_for_a_connection(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.protocol.transport = mock.Mock() - self.protocol.send_frame._is_coroutine = False # :( - - self.dispatcher = protocol.Dispatcher() - self.connection_info = ConnectionInfo('guest', 'guest', '/') - - -class LegacyOpenConnectionContext(LegacyConnectionContext): - def given_an_open_connection(self): - task = asyncio.async(open_connection(self.loop, self.protocol, self.dispatcher, self.connection_info)) - self.tick() - - start_frame = asynqp.frames.MethodFrame(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) - self.dispatcher.dispatch(start_frame) - self.tick() - - self.frame_max = 131072 - tune_frame = asynqp.frames.MethodFrame(0, spec.ConnectionTune(0, self.frame_max, 600)) - self.dispatcher.dispatch(tune_frame) - self.tick() - - open_ok_frame = asynqp.frames.MethodFrame(0, spec.ConnectionOpenOK('')) - self.dispatcher.dispatch(open_ok_frame) - self.tick() - - self.protocol.reset_mock() - self.connection = task.result() - - class ProtocolContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) - self.dispatcher = protocol.Dispatcher() + self.dispatcher = asynqp.routing.Dispatcher() self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.protocol.connection_made(self.transport) @@ -157,6 +131,6 @@ def given_a_connected_protocol(self): class MockDispatcherContext(LoopContext): def given_a_connected_protocol(self): self.transport = mock.Mock(spec=asyncio.Transport) - self.dispatcher = mock.Mock(spec=protocol.Dispatcher) + self.dispatcher = mock.Mock(spec=asynqp.routing.Dispatcher) self.protocol = protocol.AMQP(self.dispatcher, self.loop) self.protocol.connection_made(self.transport) diff --git a/test/channel_tests.py b/test/channel_tests.py index b260d33..9e716fa 100644 --- a/test/channel_tests.py +++ b/test/channel_tests.py @@ -1,8 +1,9 @@ import asyncio import contexts import asynqp +from contextlib import suppress from unittest import mock -from asynqp import spec, frames +from asynqp import spec, frames, exceptions from asynqp import message from . import util from .base_contexts import OpenConnectionContext, OpenChannelContext @@ -16,9 +17,20 @@ def it_should_send_a_channel_open_frame(self): self.server.should_have_received_method(1, spec.ChannelOpen('')) +class WhenOpeningMultipleChannelsConcurrently(OpenConnectionContext): + def when_the_user_wants_to_open_several_channels(self): + self.async_partial(asyncio.wait([self.connection.open_channel(), self.connection.open_channel()])) + + def it_should_send_a_channel_open_frame_for_channel_1(self): + self.server.should_have_received_method(1, spec.ChannelOpen('')) + + def it_should_send_a_channel_open_frame_for_channel_2(self): + self.server.should_have_received_method(2, spec.ChannelOpen('')) + + class WhenChannelOpenOKArrives(OpenConnectionContext): def given_the_user_has_called_open_channel(self): - self.task = asyncio.async(self.connection.open_channel()) + self.task = asyncio.ensure_future(self.connection.open_channel()) self.tick() def when_channel_open_ok_arrives(self): @@ -49,7 +61,7 @@ def it_should_send_ChannelClose(self): class WhenTheServerClosesAChannel(OpenChannelContext): def when_the_server_shuts_the_channel_down(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(404, 'i am tired of you', 40, 50)) def it_should_send_ChannelCloseOK(self): self.server.should_have_received_method(self.channel.id, spec.ChannelCloseOK()) @@ -69,7 +81,7 @@ def it_MUST_discard_the_method(self): class WhenAnotherMethodArrivesAfterTheServerClosedTheChannel(OpenChannelContext): def given_the_server_closed_the_channel(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(404, 'i am tired of you', 40, 50)) self.server.reset() def when_another_method_arrives(self): @@ -97,14 +109,19 @@ def it_should_not_throw_an_exception(self): class WhenAnUnexpectedChannelCloseArrives(OpenChannelContext): def given_we_are_awaiting_QueueDeclareOK(self): - self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.task = asyncio.ensure_future(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.tick() def when_ChannelClose_arrives(self): - self.server.send_method(self.channel.id, spec.ChannelClose(123, 'i am tired of you', 40, 50)) + self.server.send_method(self.channel.id, spec.ChannelClose(406, "the precondition, she failed", 50, 10)) + self.tick() def it_should_send_ChannelCloseOK(self): self.server.should_have_received_method(self.channel.id, spec.ChannelCloseOK()) + def it_should_throw_an_exception(self): + assert isinstance(self.task.exception(), exceptions.PreconditionFailed) + class WhenSettingQOS(OpenChannelContext): def when_we_are_setting_prefetch_count_only(self): @@ -116,7 +133,7 @@ def it_should_send_BasicQos_with_default_values(self): class WhenBasicQOSOkArrives(OpenChannelContext): def given_we_are_setting_qos_settings(self): - self.task = asyncio.async(self.channel.set_qos(prefetch_size=1000, prefetch_count=100, apply_globally=True)) + self.task = asyncio.ensure_future(self.channel.set_qos(prefetch_size=1000, prefetch_count=100, apply_globally=True)) self.tick() def when_BasicQosOk_arrives(self): @@ -174,7 +191,7 @@ def it_should_set_the_reply_code(self): assert self.exception.args == (self.expected_message,) def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(util.testing_exception_handler) # test that the call to handler.ready() happens at the correct time @@ -195,7 +212,7 @@ def it_should_throw_the_exception_again(self): assert self.exception is not None def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(util.testing_exception_handler) def return_msg(self): method = spec.BasicReturn(123, "you messed up", "the.exchange", "the.routing.key") @@ -209,10 +226,71 @@ def return_msg(self): self.tick() - class WhenTheHandlerIsNotCallable(OpenChannelContext): def when_I_set_the_handler(self): self.exception = contexts.catch(self.channel.set_return_handler, "i am not callable") def it_should_throw_a_TypeError(self): assert isinstance(self.exception, TypeError) + + +class WhenAConnectionIsLostCloseChannel(OpenChannelContext): + def when_connection_is_closed(self): + with suppress(Exception): + self.connection.protocol.connection_lost(Exception()) + self.tick() + self.was_closed = self.channel.is_closed() + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenWeCloseConnectionChannelShouldAlsoClose(OpenChannelContext): + def when_connection_is_closed(self): + self.task = asyncio.ensure_future(self.connection.close(), loop=self.loop) + self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() + self.was_closed = self.channel.is_closed() + self.loop.run_until_complete(asyncio.wait_for(self.task, 0.2)) + + def it_should_not_hang_channel_close(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenServerClosesConnectionChannelShouldAlsoClose(OpenChannelContext): + def when_connection_is_closed(self): + self.server.send_method( + 0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) + self.tick() + self.was_closed = self.channel.is_closed() + + def it_should_not_hang_channel_close(self): + self.loop.run_until_complete(asyncio.wait_for(self.channel.close(), 0.2)) + + def if_should_have_closed_channel(self): + assert self.was_closed + + +class WhenServerAndClientCloseChannelAtATime(OpenChannelContext): + def when_both_sides_close_channel(self): + # Client tries to close connection + self.task = asyncio.ensure_future(self.channel.close(), loop=self.loop) + self.tick() + # Before OK arrives server closes connection + self.server.send_method( + self.channel.id, + spec.ChannelClose(404, 'i am tired of you', 40, 50)) + self.tick() + self.task.result() + + def if_should_have_closed_channel(self): + assert self.channel.is_closed() + + def it_should_have_killed_synchroniser_with_404(self): + assert self.channel.synchroniser.connection_exc == exceptions.NotFound diff --git a/test/connection_tests.py b/test/connection_tests.py index 75b3d76..24a33b1 100644 --- a/test/connection_tests.py +++ b/test/connection_tests.py @@ -1,23 +1,26 @@ import asyncio import sys -import asynqp -from unittest import mock -from asynqp import spec, protocol, frames -from asynqp.connection import open_connection, ConnectionInfo -from .base_contexts import LegacyOpenConnectionContext, MockServerContext, OpenConnectionContext +from contextlib import suppress +import contexts +from asynqp import spec, exceptions +from asynqp.connection import open_connection +from .base_contexts import MockServerContext, OpenConnectionContext class WhenRespondingToConnectionStart(MockServerContext): def given_I_wrote_the_protocol_header(self): - connection_info = ConnectionInfo('guest', 'guest', '/') - self.async_partial(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} + self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) def when_ConnectionStart_arrives(self): self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) def it_should_send_start_ok(self): expected_method = spec.ConnectionStartOK( - {"product": "asynqp", "version": "0.1", "platform": sys.version}, + {"product": "asynqp", "version": "0.1", "platform": sys.version, + "capabilities": { + "consumer_cancel_notify": True + }}, 'AMQPLAIN', {'LOGIN': 'guest', 'PASSWORD': 'guest'}, 'en_US' @@ -27,8 +30,8 @@ def it_should_send_start_ok(self): class WhenRespondingToConnectionTune(MockServerContext): def given_a_started_connection(self): - connection_info = ConnectionInfo('guest', 'guest', '/') - self.async_partial(open_connection(self.loop, self.protocol, self.dispatcher, connection_info)) + connection_info = {'username': 'guest', 'password': 'guest', 'virtual_host': '/'} + self.async_partial(open_connection(self.loop, self.transport, self.protocol, self.dispatcher, connection_info)) self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) def when_ConnectionTune_arrives(self): @@ -47,6 +50,10 @@ def when_the_close_frame_arrives(self): def it_should_send_close_ok(self): self.server.should_have_received_method(0, spec.ConnectionCloseOK()) + def it_should_not_block_clonnection_close(self): + self.loop.run_until_complete( + asyncio.wait_for(self.connection.close(), 0.2)) + class WhenTheApplicationClosesTheConnection(OpenConnectionContext): def when_I_close_the_connection(self): @@ -59,50 +66,107 @@ def it_should_send_ConnectionClose_with_no_exception(self): class WhenRecievingConnectionCloseOK(OpenConnectionContext): def given_a_connection_that_I_closed(self): - asyncio.async(self.connection.close()) + asyncio.ensure_future(self.connection.close()) self.tick() def when_connection_close_ok_arrives(self): self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() def it_should_close_the_transport(self): assert self.transport.closed -# TODO: rewrite me to use a handler, not a queue writer -class WhenAConnectionThatIsClosingReceivesAMethod(LegacyOpenConnectionContext): +class WhenAConnectionThatIsClosingReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): - t = asyncio.async(self.connection.close()) + t = asyncio.ensure_future(self.connection.close()) t._log_destroy_pending = False self.tick() - - start_method = spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US') - self.start_frame = asynqp.frames.MethodFrame(0, start_method) - self.mock_writer = mock.Mock() + self.server.reset() def when_another_frame_arrives(self): - with mock.patch.dict(self.dispatcher.queue_writers, {0: self.mock_writer}): - self.dispatcher.dispatch(self.start_frame) - self.tick() + self.server.send_method(0, spec.ConnectionStart(0, 9, {}, 'PLAIN AMQPLAIN', 'en_US')) + self.tick() def it_MUST_be_discarded(self): - assert not self.mock_writer.method_calls + self.server.should_not_have_received_any() -# TODO: rewrite so it doesn't know about dispatcher -class WhenAConnectionThatWasClosedByTheServerReceivesAMethod(LegacyOpenConnectionContext): +class WhenAConnectionThatWasClosedByTheServerReceivesAMethod(OpenConnectionContext): def given_a_closed_connection(self): - close_frame = asynqp.frames.MethodFrame(0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) - self.dispatcher.dispatch(close_frame) + self.server.send_method(0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) self.tick() - self.mock_writer = mock.Mock() + self.server.reset() def when_another_frame_arrives(self): - unexpected_frame = asynqp.frames.MethodFrame(1, spec.BasicDeliver('', 1, False, '', '')) - - with mock.patch.dict(self.dispatcher.queue_writers, {1: self.mock_writer}): - self.dispatcher.dispatch(unexpected_frame) - self.tick() + self.server.send_method(0, spec.BasicDeliver('', 1, False, '', '')) + self.tick() def it_MUST_be_discarded(self): - assert not self.mock_writer.method_calls + self.server.should_not_have_received_any() + + +class WhenAConnectionIsLostCloseConnection(OpenConnectionContext): + def when_connection_is_closed(self): + with suppress(Exception): + self.connection.protocol.connection_lost(Exception()) + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) + + +class WhenServerClosesTransportWithoutConnectionClose(OpenConnectionContext): + + def given_a_channel(self): + task = self.loop.create_task(self.connection.open_channel()) + self.tick() + self.server.send_method(1, spec.ChannelOpenOK('')) + self.channel = self.wait_for(task) + + def when_server_closes_transport(self): + with suppress(exceptions.ConnectionLostError): + self.protocol.connection_lost(None) + + def it_should_raise_error_in_connection_methods(self): + try: + self.wait_for(self.channel.declare_queue("some.queue")) + except exceptions.ConnectionLostError as err: + assert type(err) == exceptions.ConnectionLostError + else: + assert False, "ConnectionLostError not raised" + + +class WhenOpeningAChannelOnAClosedConnection(OpenConnectionContext): + def when_client_closes_connection(self): + task = asyncio.ensure_future(self.connection.close()) + self.tick() + self.server.send_method(0, spec.ConnectionCloseOK()) + self.tick() + self.tick() + task.result() + + def it_should_raise_error_in_connection_methods(self): + exc = contexts.catch(self.wait_for, self.connection.open_channel()) + assert isinstance(exc, exceptions.ConnectionClosed) + + +class WhenServerAndClientCloseConnectionAtATime(OpenConnectionContext): + def when_both_sides_close_channel(self): + # Client tries to close connection + self.task = asyncio.ensure_future(self.connection.close(), loop=self.loop) + self.tick() + # Before OK arrives server closes connection + self.server.send_method( + 0, spec.ConnectionClose(123, 'you muffed up', 10, 20)) + self.tick() + self.tick() + self.task.result() + + def if_should_have_closed_connection(self): + assert self.connection._closing + + def it_should_have_killed_synchroniser_with_server_error(self): + assert isinstance( + self.connection.synchroniser.connection_exc, + exceptions.ConnectionClosed) + assert self.connection.synchroniser.connection_exc.reply_code == 123 diff --git a/test/exchange_tests.py b/test/exchange_tests.py index 3409725..64c40df 100644 --- a/test/exchange_tests.py +++ b/test/exchange_tests.py @@ -1,11 +1,11 @@ import asyncio import uuid from datetime import datetime -from unittest import mock import asynqp from asynqp import spec from asynqp import frames from asynqp import message +from asynqp import exceptions from .base_contexts import OpenChannelContext, ExchangeContext @@ -20,7 +20,7 @@ def it_should_send_ExchangeDeclare(self): class WhenExchangeDeclareOKArrives(OpenChannelContext): def given_I_declared_an_exchange(self): - self.task = asyncio.async(self.channel.declare_exchange('my.nice.exchange', 'fanout', durable=True, auto_delete=False, internal=False)) + self.task = asyncio.ensure_future(self.channel.declare_exchange('my.nice.exchange', 'fanout', durable=True, auto_delete=False, internal=False)) self.tick() def when_the_reply_arrives(self): @@ -49,7 +49,7 @@ def it_should_not_be_internal(self): class WhenIDeclareTheDefaultExchange(OpenChannelContext): def when_I_declare_an_exchange_with_an_empty_name(self): self.server.reset() - task = asyncio.async(self.channel.declare_exchange('', 'direct', durable=True, auto_delete=False, internal=False)) + task = asyncio.ensure_future(self.channel.declare_exchange('', 'direct', durable=True, auto_delete=False, internal=False)) self.tick() self.exchange = task.result() @@ -79,7 +79,7 @@ def examples_of_bad_words(cls): yield "contains'illegal$ymbols" def because_I_try_to_declare_the_exchange(self, name): - task = asyncio.async(self.channel.declare_exchange(name, 'direct')) + task = asyncio.ensure_future(self.channel.declare_exchange(name, 'direct')) self.tick() self.exception = task.exception() @@ -167,7 +167,7 @@ def it_should_send_ExchangeDelete(self): class WhenExchangeDeleteOKArrives(ExchangeContext): def given_I_deleted_the_exchange(self): - asyncio.async(self.exchange.delete(if_unused=True), loop=self.loop) + asyncio.ensure_future(self.exchange.delete(if_unused=True), loop=self.loop) self.tick() def when_confirmation_arrives(self): @@ -175,3 +175,61 @@ def when_confirmation_arrives(self): def it_should_not_throw(self): pass + + +class WhenExchangeDeclareWithPassiveAndOKArrives(OpenChannelContext): + def given_I_declared_an_exchange(self): + self.task = asyncio.ensure_future( + self.channel.declare_exchange( + 'name_1', 'fanout', passive=True, + durable=True, auto_delete=False, internal=False)) + self.tick() + + def when_the_exchange_declare_ok_arrives(self): + self.server.send_method(self.channel.id, spec.ExchangeDeclareOK()) + + def it_should_return_an_exchange_object(self): + result = self.task.result() + assert result.name == 'name_1' + assert result.type == 'fanout' + + def it_should_have_sent_passive_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.ExchangeDeclare( + 0, 'name_1', 'fanout', True, True, False, False, False, {})) + + +class WhenExchangeDeclareWithPassiveAndErrorArrives(OpenChannelContext): + def given_I_declared_an_exchange(self): + self.task = asyncio.ensure_future( + self.channel.declare_exchange( + 'name_1', 'fanout', passive=True, + durable=True, auto_delete=False, internal=False)) + self.tick() + + def when_error_arrives(self): + self.server.send_method( + self.channel.id, spec.ChannelClose(404, 'Bad exchange', 40, 50)) + + def it_should_raise_not_found_error(self): + assert isinstance(self.task.exception(), exceptions.NotFound) + + +class WhenIDeclareExchangeWithNoWait(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.ensure_future(self.channel.declare_exchange( + 'my.nice.exchange', 'fanout', durable=True, auto_delete=False, + internal=False, nowait=True), loop=self.loop) + self.tick() + + def it_should_return_exchange_object_without_wait(self): + result = self.task.result() + assert result + assert result.name == 'my.nice.exchange' + assert result.type == 'fanout' + + def it_should_have_sent_nowait_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.ExchangeDeclare( + 0, 'my.nice.exchange', 'fanout', False, True, False, False, + True, {})) diff --git a/test/heartbeat_tests.py b/test/heartbeat_tests.py index a97c0b1..7a6965e 100644 --- a/test/heartbeat_tests.py +++ b/test/heartbeat_tests.py @@ -1,71 +1,60 @@ +import asyncio from unittest import mock -from asynqp import spec -from asynqp import frames -from asynqp import protocol -from .base_contexts import ProtocolContext, MockLoopContext +from asynqp.frames import HeartbeatFrame +from asynqp.exceptions import ConnectionLostError +from .base_contexts import MockServerContext -class WhenStartingTheHeartbeat(ProtocolContext, MockLoopContext): - def when_I_start_the_heartbeat(self): - self.protocol.start_heartbeat(5) +class WhenServerWaitsForHeartbeat(MockServerContext): + def when_heartbeating_starts(self): + self.protocol.start_heartbeat(0.01) + self.loop.run_until_complete(asyncio.sleep(0.015)) - def it_should_set_up_heartbeat_and_timeout_callbacks(self): - assert self.loop.call_later.call_args_list == [ - mock.call(5, self.protocol.heartbeat_monitor.send_heartbeat), - mock.call(10, self.protocol.heartbeat_monitor.heartbeat_timed_out) - ] + def it_should_send_the_heartbeat(self): + self.server.should_have_received_frame(HeartbeatFrame()) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) -class WhenHeartbeatIsDisabled(ProtocolContext, MockLoopContext): - def given_the_server_does_not_want_a_heartbeat(self): - self.heartbeat_interval = 0 - def when_I_start_the_heartbeat(self): - self.protocol.start_heartbeat(self.heartbeat_interval) +class WhenServerRespondsToHeartbeat(MockServerContext): + def given_i_started_heartbeating(self): + self.protocol.start_heartbeat(0.01) + self.loop.run_until_complete(asyncio.sleep(0.015)) - def it_should_not_set_up_callbacks(self): - assert not self.loop.call_later.called + def when_the_server_replies(self): + self.server.send_frame(HeartbeatFrame()) + self.loop.run_until_complete(asyncio.sleep(0.005)) + def it_should_send_the_heartbeat(self): + self.server.should_have_received_frames([HeartbeatFrame(), HeartbeatFrame()]) -class WhenItIsTimeToHeartbeat(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) - def when_the_event_loop_comes_a_knockin(self): - self.heartbeat_monitor.send_heartbeat() - def it_should_send_a_heartbeat_frame(self): - self.protocol.send_frame.assert_called_once_with(frames.HeartbeatFrame()) +class WhenServerDoesNotRespondToHeartbeat(MockServerContext): + def given_i_started_heartbeating(self): + self.protocol.start_heartbeat(0.01) - def it_should_set_up_the_next_heartbeat(self): - self.loop.call_later.assert_called_once_with(5, self.heartbeat_monitor.send_heartbeat) + def when_the_server_dies(self): + with mock.patch("asynqp.routing.Dispatcher.dispatch_all") as mocked: + self.loop.run_until_complete(asyncio.sleep(0.021)) + self.mocked = mocked + def it_should_dispatch_a_poison_pill(self): + assert self.mocked.called + assert isinstance( + self.mocked.call_args[0][0].exception, ConnectionLostError) -class WhenResettingTheHeartbeatTimeout(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) - self.heartbeat_monitor.monitor_heartbeat() - self.loop.reset_mock() - - def because_the_timeout_gets_reset(self): - self.heartbeat_monitor.heartbeat_received() - - def it_should_cancel_the_close_callback(self): - self.loop.call_later.return_value.cancel.assert_called_once_with() - - def it_should_set_up_another_close_callback(self): - self.loop.call_later.assert_called_once_with(10, self.heartbeat_monitor.heartbeat_timed_out) - - -class WhenTheHeartbeatTimesOut(MockLoopContext): - def given_a_heartbeat_monitor(self): - self.protocol = mock.Mock(spec=protocol.AMQP) - self.heartbeat_monitor = protocol.HeartbeatMonitor(self.protocol, self.loop, 5) - - def when_the_heartbeat_times_out(self): - self.heartbeat_monitor.heartbeat_timed_out() - - def it_should_send_connection_close(self): - self.protocol.send_method.assert_called_once_with(0, spec.ConnectionClose(501, 'Heartbeat timed out', 0, 0)) + def cleanup_tasks(self): + self.protocol.heartbeat_monitor.stop() + self.loop.run_until_complete( + asyncio.wait_for(self.protocol.heartbeat_monitor.wait_closed(), + timeout=0.2)) diff --git a/test/integration_tests.py b/test/integration_tests.py index 80f581c..1aa1f80 100644 --- a/test/integration_tests.py +++ b/test/integration_tests.py @@ -1,6 +1,8 @@ import asyncio import asynqp +import socket import contexts +from .util import run_briefly class ConnectionContext: @@ -22,10 +24,10 @@ def cleanup_the_channel(self): class BoundQueueContext(ChannelContext): def given_a_queue_bound_to_an_exchange(self): - self.loop.run_until_complete(asyncio.wait_for(self.setup(), 0.4)) + self.loop.run_until_complete(asyncio.wait_for(self.setup(), 0.5)) def cleanup_the_queue_and_exchange(self): - self.loop.run_until_complete(asyncio.wait_for(self.teardown(), 0.2)) + self.loop.run_until_complete(asyncio.wait_for(self.teardown(), 0.3)) @asyncio.coroutine def setup(self): @@ -54,6 +56,22 @@ def cleanup_the_connection(self): self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) +class WhenConnectingToRabbitWithAnExistingSocket: + def given_the_loop(self): + self.loop = asyncio.get_event_loop() + self.sock = socket.create_connection(("localhost", 5672)) + + def when_I_connect(self): + self.connection = self.loop.run_until_complete(asyncio.wait_for(asynqp.connect(sock=self.sock), 0.2)) + + def it_should_connect(self): + assert self.connection is not None + + def cleanup_the_connection(self): + self.loop.run_until_complete(asyncio.wait_for(self.connection.close(), 0.2)) + self.sock.close() + + class WhenOpeningAChannel(ConnectionContext): def when_I_open_a_channel(self): self.channel = self.loop.run_until_complete(asyncio.wait_for(self.connection.open_channel(), 0.2)) @@ -66,12 +84,18 @@ def cleanup_the_channel(self): class WhenDeclaringAQueue(ChannelContext): + ARGUMENTS = {'x-expires': 300, 'x-message-ttl': 1000, 'x-table-test': {'a': [1, 'a', {}, []], 'c': 1}} + def when_I_declare_a_queue(self): - self.queue = self.loop.run_until_complete(asyncio.wait_for(self.channel.declare_queue('my.queue', exclusive=True), 0.2)) + coro = self.channel.declare_queue('my.queue', exclusive=True, arguments=WhenDeclaringAQueue.ARGUMENTS) + self.queue = self.loop.run_until_complete(asyncio.wait_for(coro, 0.2)) def it_should_have_the_correct_queue_name(self): assert self.queue.name == 'my.queue' + def it_should_have_the_correct_attributes_in_rabbitmq(self): + assert self.queue.arguments == WhenDeclaringAQueue.ARGUMENTS + def cleanup_the_queue(self): self.loop.run_until_complete(asyncio.wait_for(self.queue.delete(if_unused=False, if_empty=False), 0.2)) @@ -154,7 +178,7 @@ def start_consumer(self): class WhenPublishingAndGettingALongMessage(BoundQueueContext): def given_a_multi_frame_message_and_a_consumer(self): - frame_max = self.connection.connection_info.frame_max + frame_max = self.connection.connection_info['frame_max'] body1 = "a" * (frame_max - 8) body2 = "b" * (frame_max - 8) body3 = "c" * (frame_max - 8) @@ -171,7 +195,7 @@ def it_should_return_my_message(self): class WhenPublishingAndConsumingALongMessage(BoundQueueContext): def given_a_multi_frame_message(self): - frame_max = self.connection.connection_info.frame_max + frame_max = self.connection.connection_info['frame_max'] body1 = "a" * (frame_max - 8) body2 = "b" * (frame_max - 8) body3 = "c" * (frame_max - 8) @@ -199,3 +223,101 @@ def when_I_cancel_the_consumer_and_also_get_a_message(self): def it_should_not_throw(self): assert self.exception is None + + +class WhenPublishingWithUnsetLoop: + + def given_I_have_a_queue(self): + @asyncio.coroutine + def set_up(): + self.connection = yield from asynqp.connect(loop=self.loop) + self.channel = yield from self.connection.open_channel() + self.exchange = yield from self.channel.declare_exchange( + '', 'direct') + self.queue = yield from self.channel.declare_queue( + durable=False, + exclusive=True, + auto_delete=True) + self.loop = asyncio.get_event_loop() + asyncio.set_event_loop(None) + self.loop.run_until_complete(set_up()) + + def when_I_publish_the_message(self): + message = asynqp.Message(b"Test message") + self.exchange.publish(message, self.queue.name) + + def it_should_return_my_message(self): + for retry in range(10): + msg = self.loop.run_until_complete(self.queue.get(no_ack=True)) + if msg is not None: + break + assert msg.body == b"Test message" + + def cleanup_loop(self): + @asyncio.coroutine + def tear_down(): + yield from self.channel.close() + yield from self.connection.close() + self.loop.run_until_complete(tear_down()) + asyncio.set_event_loop(self.loop) + + +class WhenConsumingWithUnsetLoop: + + def given_I_published_a_message(self): + @asyncio.coroutine + def set_up(): + self.connection = yield from asynqp.connect(loop=self.loop) + self.channel = yield from self.connection.open_channel() + self.exchange = yield from self.channel.declare_exchange( + '', 'direct') + self.queue = yield from self.channel.declare_queue( + durable=False, + exclusive=True, + auto_delete=True) + self.loop = asyncio.get_event_loop() + asyncio.set_event_loop(None) + self.loop.run_until_complete(set_up()) + + message = asynqp.Message(b"Test message") + self.exchange.publish(message, self.queue.name) + + def when_I_consume_a_message(self): + self.result = [] + consumer = self.loop.run_until_complete( + self.queue.consume(self.result.append, exclusive=True)) + for retry in range(10): + run_briefly(self.loop) + if self.result: + break + consumer.cancel() + + def it_should_return_my_message(self): + assert self.result, "Message not consumed" + assert self.result[0].body == b"Test message" + + def cleanup_loop(self): + @asyncio.coroutine + def tear_down(): + yield from self.channel.close() + yield from self.connection.close() + self.loop.run_until_complete(tear_down()) + asyncio.set_event_loop(self.loop) + + +class WhenISendZeroMessage(BoundQueueContext): + def given_an_empty_message(self): + self.message = asynqp.Message('') + self.exchange.publish(self.message, 'routingkey') + + def when_I_start_a_consumer(self): + self.message_received = asyncio.Future() + self.loop.run_until_complete(asyncio.wait_for(self.start_consumer(), 0.2)) + + def it_should_deliver_the_message_to_the_consumer(self): + assert self.message_received.result() == self.message + + @asyncio.coroutine + def start_consumer(self): + yield from self.queue.consume(self.message_received.set_result) + yield from self.message_received diff --git a/test/message_tests.py b/test/message_tests.py index 48223ef..13fb050 100644 --- a/test/message_tests.py +++ b/test/message_tests.py @@ -3,6 +3,7 @@ import uuid from datetime import datetime import asynqp +from asynqp import amqptypes from asynqp import message from asynqp import spec from asynqp import frames @@ -127,7 +128,7 @@ def given_I_received_a_message(self): self.delivery_tag = 12487 msg = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - task = asyncio.async(self.queue.get()) + task = asyncio.ensure_future(self.queue.get()) self.tick() method = spec.BasicGetOK(self.delivery_tag, False, 'my.exchange', 'routing.key', 0) self.server.send_method(self.channel.id, method) @@ -153,7 +154,7 @@ def given_I_received_a_message(self): self.delivery_tag = 12487 msg = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - task = asyncio.async(self.queue.get()) + task = asyncio.ensure_future(self.queue.get()) self.tick() method = spec.BasicGetOK(self.delivery_tag, False, 'my.exchange', 'routing.key', 0) self.server.send_method(self.channel.id, method) @@ -184,3 +185,54 @@ def when_I_get_the_json(self): def it_should_give_me_the_body(self): assert self.result == self.body + + +class WhenSettingAProperty: + def given_a_message(self): + self.msg = asynqp.Message("abc") + + def when_I_set_a_property(self): + self.msg.content_type = "application/json" + + def it_should_cast_it_to_the_correct_amqp_type(self): + assert isinstance(self.msg.content_type, amqptypes.ShortStr) + assert self.msg.content_type == amqptypes.ShortStr("application/json") + + +class WhenSettingAPropertyAndIHaveAlreadyCastItMyself: + def given_a_message(self): + self.msg = asynqp.Message("abc") + self.val = amqptypes.ShortStr("application/json") + + def when_I_set_a_property(self): + self.msg.content_type = self.val + + def it_should_not_attempt_to_cast_it(self): + assert self.msg.content_type is self.val + + +class WhenSettingAnAttributeThatIsNotAProperty: + def given_a_message(self): + self.msg = asynqp.Message("abc") + + def when_I_set_a_property(self): + self.msg.foo = 123 + + def it_should_not_attempt_to_cast_it(self): + assert self.msg.foo == 123 + + +class WhenIReadAContentHeaderWithoutAllProperties: + + def given_headers_wit_only_content_encoding(self): + self.data = ( + b'\x00<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08p\x00' + b'\x05utf-8\x00\x00\x00\x00\x01') + + def when_I_read_properties(self): + self.payload = message.ContentHeaderPayload.read(self.data) + + def it_should_have_only_content_encoding(self): + assert self.payload == message.ContentHeaderPayload( + 60, 8, [None, 'utf-8', {}, 1, None, None, + None, None, None, None, None, None, None]) diff --git a/test/method_tests.py b/test/method_tests.py index 79bc83e..5c0fc23 100644 --- a/test/method_tests.py +++ b/test/method_tests.py @@ -1,5 +1,4 @@ import asynqp -from unittest import mock from asynqp import spec from asynqp import frames from asynqp import amqptypes @@ -165,4 +164,3 @@ def when_the_frame_arrives(self): def it_should_deserialise_it_to_the_correct_method(self): self.dispatcher.dispatch.assert_called_once_with(self.expected_frame) - diff --git a/test/protocol_tests.py b/test/protocol_tests.py index 841843d..e9ef554 100644 --- a/test/protocol_tests.py +++ b/test/protocol_tests.py @@ -3,7 +3,9 @@ import asynqp from asynqp import spec from asynqp import protocol +from asynqp.exceptions import ConnectionLostError from .base_contexts import MockDispatcherContext, MockServerContext +from .util import testing_exception_handler class WhenInitiatingProceedings(MockServerContext): @@ -130,3 +132,26 @@ def because_two_frames_arrive_in_bits(self, fragments): def it_should_dispatch_the_method_twice(self): self.dispatcher.dispatch.assert_has_calls([mock.call(self.expected_frame), mock.call(self.expected_frame)]) + + +class WhenTheConnectionIsLost(MockServerContext): + def given_an_exception_handler(self): + self.connection_lost_error_raised = False + self.loop.set_exception_handler(self.exception_handler) + + def exception_handler(self, loop, context): + exception = context.get('exception') + if type(exception) is ConnectionLostError: + self.connection_lost_error_raised = True + else: + self.loop.default_exception_handler(context) + + def when_the_connection_is_closed(self): + self.loop.call_soon(self.protocol.connection_lost, Exception) + self.tick() + + def it_should_raise_a_connection_lost_error(self): + assert self.connection_lost_error_raised is True + + def cleanup(self): + self.loop.set_exception_handler(testing_exception_handler) diff --git a/test/queue_tests.py b/test/queue_tests.py index b2753c3..91cbab4 100644 --- a/test/queue_tests.py +++ b/test/queue_tests.py @@ -1,26 +1,29 @@ import asyncio from datetime import datetime -from unittest import mock +from contextlib import suppress +import contexts import asynqp from asynqp import message from asynqp import frames from asynqp import spec +from asynqp import exceptions from .base_contexts import OpenChannelContext, QueueContext, ExchangeContext, BoundQueueContext, ConsumerContext +from .util import testing_exception_handler class WhenDeclaringAQueue(OpenChannelContext): def when_I_declare_a_queue(self): - self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True)) + self.async_partial(self.channel.declare_queue('my.nice.queue', durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) def it_should_send_a_QueueDeclare_method(self): - expected_method = spec.QueueDeclare(0, 'my.nice.queue', False, True, True, True, False, {}) + expected_method = spec.QueueDeclare(0, 'my.nice.queue', False, True, True, True, False, {'x-expires': 300, 'x-message-ttl': 1000}) self.server.should_have_received_method(self.channel.id, expected_method) class WhenQueueDeclareOKArrives(OpenChannelContext): def given_I_declared_a_queue(self): self.queue_name = 'my.nice.queue' - self.task = asyncio.async(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True)) + self.task = asyncio.ensure_future(self.channel.declare_queue(self.queue_name, durable=True, exclusive=True, auto_delete=True, arguments={'x-expires': 300, 'x-message-ttl': 1000})) self.tick() def when_QueueDeclareOK_arrives(self): @@ -42,7 +45,7 @@ def it_should_auto_delete(self): class WhenILetTheServerPickTheQueueName(OpenChannelContext): def given_I_declared_a_queue(self): - self.task = asyncio.async(self.channel.declare_queue('', durable=True, exclusive=True, auto_delete=True), loop=self.loop) + self.task = asyncio.ensure_future(self.channel.declare_queue('', durable=True, exclusive=True, auto_delete=True), loop=self.loop) self.tick() self.queue_name = 'randomly.generated.name' @@ -62,8 +65,8 @@ def examples_of_bad_names(cls): yield 'contains~illegal/symbols' def when_I_declare_the_queue(self, queue_name): - self.task = asyncio.async(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), - loop=self.loop) + self.task = asyncio.ensure_future(self.channel.declare_queue(queue_name, durable=True, exclusive=True, auto_delete=True), + loop=self.loop) self.tick() def it_should_throw_ValueError(self): @@ -72,16 +75,24 @@ def it_should_throw_ValueError(self): class WhenBindingAQueueToAnExchange(QueueContext, ExchangeContext): def when_I_bind_the_queue(self): - self.async_partial(self.queue.bind(self.exchange, 'routing.key')) + self.async_partial(self.queue.bind(self.exchange, 'routing.key', arguments={'x-ignore': ''})) def it_should_send_QueueBind(self): - expected_method = spec.QueueBind(0, self.queue.name, self.exchange.name, 'routing.key', False, {}) + expected_method = spec.QueueBind(0, self.queue.name, self.exchange.name, 'routing.key', False, {'x-ignore': ''}) self.server.should_have_received_method(self.channel.id, expected_method) +class WhenBindingAQueueToTheDefaultExchange(QueueContext): + def when_I_bind_the_queue(self): + self.task = self.async_partial(self.queue.bind('', 'routing.key', arguments={'x-ignore': ''})) + + def it_should_throw_InvalidExchangeName(self): + assert isinstance(self.task.exception(), exceptions.InvalidExchangeName) + + class WhenQueueBindOKArrives(QueueContext, ExchangeContext): def given_I_sent_QueueBind(self): - self.task = asyncio.async(self.queue.bind(self.exchange, 'routing.key')) + self.task = asyncio.ensure_future(self.queue.bind(self.exchange, 'routing.key')) self.tick() def when_QueueBindOK_arrives(self): @@ -97,16 +108,16 @@ def and_the_returned_binding_should_have_the_correct_exchange(self): class WhenUnbindingAQueue(BoundQueueContext): def when_I_unbind_the_queue(self): - self.async_partial(self.binding.unbind()) + self.async_partial(self.binding.unbind(arguments={'x-ignore': ''})) def it_should_send_QueueUnbind(self): - expected_method = spec.QueueUnbind(0, self.queue.name, self.exchange.name, 'routing.key', {}) + expected_method = spec.QueueUnbind(0, self.queue.name, self.exchange.name, 'routing.key', {'x-ignore': ''}) self.server.should_have_received_method(self.channel.id, expected_method) class WhenQueueUnbindOKArrives(BoundQueueContext): def given_I_unbound_the_queue(self): - self.task = asyncio.async(self.binding.unbind()) + self.task = asyncio.ensure_future(self.binding.unbind()) self.tick() def when_QueueUnbindOK_arrives(self): @@ -118,12 +129,12 @@ def it_should_be_ok(self): class WhenIUnbindAQueueTwice(BoundQueueContext): def given_an_unbound_queue(self): - asyncio.async(self.binding.unbind()) + asyncio.ensure_future(self.binding.unbind()) self.tick() self.server.send_method(self.channel.id, spec.QueueUnbindOK()) def when_I_unbind_the_queue_again(self): - self.task = asyncio.async(self.binding.unbind()) + self.task = asyncio.ensure_future(self.binding.unbind()) self.tick() def it_should_throw_Deleted(self): @@ -140,7 +151,7 @@ def it_should_send_BasicGet(self): class WhenBasicGetEmptyArrives(QueueContext): def given_I_asked_for_a_message(self): - self.task = asyncio.async(self.queue.get(no_ack=False)) + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) self.tick() def when_BasicGetEmpty_arrives(self): @@ -153,7 +164,7 @@ def it_should_return_None(self): class WhenBasicGetOKArrives(QueueContext): def given_I_asked_for_a_message(self): self.expected_message = asynqp.Message('body', timestamp=datetime(2014, 5, 5)) - self.task = asyncio.async(self.queue.get(no_ack=False)) + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) self.tick() def when_BasicGetOK_arrives_with_content(self): @@ -177,17 +188,32 @@ def it_should_put_the_routing_key_on_the_msg(self): assert self.task.result().routing_key == 'routing.key' +class WhenConnectionClosedOnGet(QueueContext): + def given_I_asked_for_a_message(self): + self.task = asyncio.ensure_future(self.queue.get(no_ack=False)) + self.tick() + + def when_connection_is_closed(self): + # XXX: remove if we change behaviour to not raise + with suppress(Exception): + self.server.protocol.connection_lost(Exception()) + self.tick() + + def it_should_raise_exception(self): + assert self.task.exception() is not None + + class WhenISubscribeToAQueue(QueueContext): def when_I_start_a_consumer(self): - self.async_partial(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False)) + self.async_partial(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) def it_should_send_BasicConsume(self): - self.server.should_have_received_method(self.channel.id, spec.BasicConsume(0, self.queue.name, '', False, False, False, False, {})) + self.server.should_have_received_method(self.channel.id, spec.BasicConsume(0, self.queue.name, '', False, False, False, False, {'x-priority': 1})) class WhenConsumeOKArrives(QueueContext): def given_I_started_a_consumer(self): - self.task = asyncio.async(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False)) + self.task = asyncio.ensure_future(self.queue.consume(lambda msg: None, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) self.tick() def when_BasicConsumeOK_arrives(self): @@ -244,7 +270,7 @@ def deliver_msg(self): self.tick() def cleanup_the_exception_handler(self): - self.loop.set_exception_handler(None) + self.loop.set_exception_handler(testing_exception_handler) class WhenICancelAConsumer(ConsumerContext): @@ -257,7 +283,7 @@ def it_should_send_a_BasicCancel_method(self): class WhenCancelOKArrives(ConsumerContext): def given_I_cancelled_a_consumer(self): - asyncio.async(self.consumer.cancel()) + asyncio.ensure_future(self.consumer.cancel()) self.tick() def when_BasicCancelOK_arrives(self): @@ -267,6 +293,60 @@ def it_should_be_cancelled(self): assert self.consumer.cancelled +class WhenCancelOKArrivesForAConsumerWithAnOnCancelMethod(QueueContext): + def given_I_started_and_cancelled_a_consumer(self): + self.consumer = self.ConsumerWithOnCancel() + task = asyncio.ensure_future(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + asyncio.ensure_future(task.result().cancel()) + self.tick() + + def when_BasicCancelOK_arrives(self): + self.server.send_method(self.channel.id, spec.BasicCancelOK('made.up.tag')) + + def it_should_call_on_cancel(self): + assert self.consumer.on_cancel_called + + class ConsumerWithOnCancel: + def __init__(self): + self.on_cancel_called = False + + def __call__(self): + pass + + def on_cancel(self): + self.on_cancel_called = True + + +class WhenAConsumerWithAnOnCancelMethodIsKilledDueToAnError(QueueContext): + def given_I_started_a_consumer(self): + self.consumer = self.ConsumerWithOnError() + asyncio.ensure_future(self.queue.consume(self.consumer, no_local=False, no_ack=False, exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.exception = Exception() + + def when_the_connection_dies(self): + contexts.catch(self.protocol.connection_lost, self.exception) + self.tick() + + def it_should_call_on_error(self): + assert self.consumer.exc.original_exc is self.exception + + class ConsumerWithOnError: + def __init__(self): + self.exc = None + + def __call__(self): + pass + + def on_error(self, exc): + self.exc = exc + + class WhenIPurgeAQueue(QueueContext): def because_I_purge_the_queue(self): self.async_partial(self.queue.purge()) @@ -277,7 +357,7 @@ def it_should_send_a_QueuePurge_method(self): class WhenQueuePurgeOKArrives(QueueContext): def given_I_called_queue_purge(self): - self.task = asyncio.async(self.queue.purge()) + self.task = asyncio.ensure_future(self.queue.purge()) self.tick() def when_QueuePurgeOK_arrives(self): @@ -297,7 +377,7 @@ def it_should_send_a_QueueDelete_method(self): class WhenQueueDeleteOKArrives(QueueContext): def given_I_deleted_a_queue(self): - asyncio.async(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) + asyncio.ensure_future(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) self.tick() def when_QueueDeleteOK_arrives(self): @@ -310,13 +390,102 @@ def it_should_be_deleted(self): class WhenITryToUseADeletedQueue(QueueContext): def given_a_deleted_queue(self): - asyncio.async(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) + asyncio.ensure_future(self.queue.delete(if_unused=False, if_empty=False), loop=self.loop) self.tick() self.server.send_method(self.channel.id, spec.QueueDeleteOK(123)) def when_I_try_to_use_the_queue(self): - self.task = asyncio.async(self.queue.get()) + self.task = asyncio.ensure_future(self.queue.get()) self.tick() def it_should_throw_Deleted(self): assert isinstance(self.task.exception(), asynqp.Deleted) + + +class WhenAConnectionIsClosedCancelConsuming(QueueContext, ExchangeContext): + def given_a_consumer(self): + task = asyncio.ensure_future(self.queue.consume( + lambda x: None, no_local=False, no_ack=False, + exclusive=False, arguments={'x-priority': 1})) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.consumer = task.result() + + def when_connection_is_closed(self): + with suppress(Exception): + self.connection.protocol.connection_lost(Exception()) + + def it_should_not_hang(self): + self.loop.run_until_complete(asyncio.wait_for(self.consumer.cancel(), 0.2)) + + +class WhenIDeclareQueueWithPassiveAndOKArrives(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.ensure_future(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=False, + passive=True), loop=self.loop) + self.tick() + + def when_QueueDeclareOK_arrives(self): + self.server.send_method( + self.channel.id, spec.QueueDeclareOK('123', 123, 456)) + + def it_should_return_queue_object(self): + result = self.task.result() + assert result + assert result.name == '123' + + def it_should_have_sent_passive_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.QueueDeclare( + 0, '123', True, True, True, False, False, {})) + + +class WhenIDeclareQueueWithPassiveAndErrorArrives(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.ensure_future(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=True, + passive=True), loop=self.loop) + self.tick() + + def when_error_arrives(self): + self.server.send_method( + self.channel.id, spec.ChannelClose(404, 'Bad queue', 40, 50)) + + def it_should_raise_exception(self): + assert isinstance(self.task.exception(), exceptions.NotFound) + + +class WhenIDeclareQueueWithNoWait(OpenChannelContext): + def given_I_declared_a_queue_with_passive(self): + self.task = asyncio.ensure_future(self.channel.declare_queue( + '123', durable=True, exclusive=True, auto_delete=False, + nowait=True), loop=self.loop) + self.tick() + + def it_should_return_queue_object_without_wait(self): + result = self.task.result() + assert result + assert result.name == '123' + + def it_should_have_sent_nowait_in_frame(self): + self.server.should_have_received_method( + self.channel.id, spec.QueueDeclare( + 0, '123', False, True, True, False, True, {})) + + +class WhenConsumerIsClosedServerSide(QueueContext, ExchangeContext): + def given_a_consumer(self): + task = asyncio.ensure_future(self.queue.consume(lambda x: None)) + self.tick() + self.server.send_method(self.channel.id, spec.BasicConsumeOK('made.up.tag')) + self.tick() + self.consumer = task.result() + + def when_consumer_is_closed_server_side(self): + self.server.send_method(self.channel.id, spec.BasicCancel('made.up.tag', False)) + self.tick() + + def it_should_close_consumer(self): + assert 'made.up.tag' not in self.channel.queue_factory.consumers.consumers diff --git a/test/serialisation_tests.py b/test/serialisation_tests.py index 551afc7..1b674ca 100644 --- a/test/serialisation_tests.py +++ b/test/serialisation_tests.py @@ -1,13 +1,21 @@ from io import BytesIO import contexts +from datetime import datetime, timezone, timedelta from asynqp import serialisation, AMQPError class WhenParsingATable: @classmethod def examples_of_tables(self): + yield b"\x00\x00\x00\x00", {} yield b"\x00\x00\x00\x0E\x04key1t\x00\x04key2t\x01", {'key1': False, 'key2': True} - yield b"\x00\x00\x00\x0B\x03keys\x05hello", {'key': 'hello'} + yield b"\x00\x00\x00\x06\x03keyb\xff", {'key': -1} + yield b"\x00\x00\x00\x07\x03keys\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x09\x03keyI\xff\xff\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x0C\x03keyl\xff\xff\xff\xff\xff\xff\xff\xff", {'key': -1} + yield b"\x00\x00\x00\x05\x03keyV", {'key': None} + yield b"\x00\x00\x00\x05\x03keyA\x00\x00\x00\x00", {'key': []} + yield b"\x00\x00\x00\x0C\x03keyx\x00\x00\x00\x04\x00\x01\x02\x03", {'key': b"\x00\x01\x02\x03"} yield b"\x00\x00\x00\x0E\x03keyS\x00\x00\x00\x05hello", {'key': 'hello'} yield b"\x00\x00\x00\x16\x03keyF\x00\x00\x00\x0D\x0Aanotherkeyt\x00", {'key': {'anotherkey': False}} @@ -18,6 +26,39 @@ def it_should_return_the_table(self, bytes, expected): assert self.result == expected +class WhenPackingAndUnpackingATable: + @classmethod + def examples_of_tables(cls): + yield {'a': (1 << 16), 'b': (1 << 15)} + yield {'c': 65535, 'd': -65535} + yield {'e': -65536} + yield {'f': -0x7FFFFFFF, 'g': 0x7FFFFFFF} + yield {'x': b"\x01\x02"} + yield {'x': []} + yield {'l': None} + yield {'l': 1.0} + + def because_we_pack_and_unpack_the_table(self, table): + self.result = serialisation.read_table(BytesIO(serialisation.pack_table(table))) + + def it_should_return_the_table(self, table): + assert self.result == table + + +class WhenParsingAHugeTable: + @classmethod + def examples_of_huge_tables(self): + # That would be -1 for an signed int + yield b"\xFF\xFF\xFF\xFF\xFF" + + def because_we_read_the_table(self, bytes): + # We expect the serialisation to read over the bounds, but only if it is unsigned + self.exception = contexts.catch(serialisation.read_table, BytesIO(bytes)) + + def it_should_throw_an_AMQPError(self): + assert isinstance(self.exception, AMQPError) + + class WhenParsingABadTable: @classmethod def examples_of_bad_tables(self): @@ -31,6 +72,28 @@ def it_should_throw_an_AMQPError(self): assert isinstance(self.exception, AMQPError) +class WhenParsingAnArray: + @classmethod + def examples_of_arrays(self): + yield b"\x00\x00\x00\x00", [] + yield b"\x00\x00\x00\x04t\x00t\x01", [False, True] + yield b"\x00\x00\x00\x02b\xff", [-1] + yield b"\x00\x00\x00\x03s\xff\xff", [-1] + yield b"\x00\x00\x00\x05I\xff\xff\xff\xff", [-1] + yield b"\x00\x00\x00\x09l\xff\xff\xff\xff\xff\xff\xff\xff", [-1] + yield b"\x00\x00\x00\x01V", [None] + yield b"\x00\x00\x00\x05A\x00\x00\x00\x00", [[]] + yield b"\x00\x00\x00\x09x\x00\x00\x00\x04\x00\x01\x02\x03", [b"\x00\x01\x02\x03"] + yield b"\x00\x00\x00\x0AS\x00\x00\x00\x05hello", ['hello'] + yield b"\x00\x00\x00\x12F\x00\x00\x00\x0D\x0Aanotherkeyt\x00", [{'anotherkey': False}] + + def because_we_read_the_array(self, buffer, expected): + self.result = serialisation.read_array(BytesIO(buffer)) + + def it_should_return_the_array(self, buffer, expected): + assert self.result == expected + + class WhenParsingALongString: def because_we_read_a_long_string(self): self.result = serialisation.read_long_string(BytesIO(b"\x00\x00\x00\x05hello")) @@ -61,3 +124,51 @@ def because_I_pack_them(self, bools, expected): def it_should_pack_them_correctly(self, bools, expected): assert self.result == expected + + +class WhenParsingATimestamp: + @classmethod + def examples_of_timestamps(cls): + # The timestamp should be zero relative to epoch + yield b'\x00\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, tzinfo=timezone.utc) + # And independent of the timezone + yield b'\x00\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, 1, 30, tzinfo=timezone(timedelta(hours=1, minutes=30))) + # And and increase by a millisecond + yield b'\x00\x00\x00\x00\x00\x00\x00\x01', datetime(1970, 1, 1, microsecond=1000, tzinfo=timezone.utc) + # Cannot validate, that it is unsigned, as it is + # yield b'\x80\x00\x00\x00\x00\x00\x00\x00', datetime(1970, 1, 1, microsecond=1000, tzinfo=timezone.utc) + + def because_we_read_a_timestamp(self, binary, _): + self.result = serialisation.read_timestamp(BytesIO(binary)) + + def it_should_read_it_correctly(self, _, expected): + assert self.result == expected + + +class WhenWritingATimestamp: + @classmethod + def examples_of_timestamps(cls): + for encoded, timeval in WhenParsingATimestamp.examples_of_timestamps(): + yield timeval, encoded + + def because_I_pack_them(self, timeval, _): + self.result = serialisation.pack_timestamp(timeval) + + def it_should_pack_them_correctly(self, _, expected): + assert self.result == expected + + +class WhenPackingAndUnpackingATimestamp: + # Ensure, we do not add some offset by the serialisation process + @classmethod + def examples_of_timestamps(cls): + yield datetime(1970, 1, 1, tzinfo=timezone.utc) + yield datetime(1979, 1, 1, tzinfo=timezone(timedelta(hours=1, minutes=30))) + + def because_I_pack_them(self, timeval): + packed = serialisation.pack_timestamp(timeval) + unpacked = serialisation.read_timestamp(BytesIO(packed)) + self.result = unpacked - timeval + + def it_should_pack_them_correctly(self, timeval): + assert abs(self.result.total_seconds()) < 1.0e-9 diff --git a/test/util.py b/test/util.py index bfea180..aa6b1f9 100644 --- a/test/util.py +++ b/test/util.py @@ -1,108 +1,137 @@ -import asyncio -from unittest import mock -import asynqp.frames -from asynqp import protocol - - -class MockServer(object): - def __init__(self, protocol, tick): - self.protocol = protocol - self.tick = tick - self.data = [] - - def send_bytes(self, b): - self.protocol.data_received(b) - self.tick() - - def send_frame(self, frame): - self.send_bytes(frame.serialise()) - - def send_method(self, channel_number, method): - frame = asynqp.frames.MethodFrame(channel_number, method) - self.send_frame(frame) - - def reset(self): - self.data = [] - - def should_have_received_frames(self, expected_frames, any_order=False): - results = (read(x) for x in self.data) - frames = [x for x in results if x is not None] - if any_order: - for frame in expected_frames: - assert frame in frames, "{} should have been in {}".format(frame, frames) - else: - expected_frames = tuple(expected_frames) - assert expected_frames in windows(frames, len(expected_frames)), "{} should have been in {}".format(expected_frames, frames) - - def should_have_received_methods(self, channel_number, methods, any_order=False): - frames = (asynqp.frames.MethodFrame(channel_number, m) for m in methods) - self.should_have_received_frames(frames, any_order) - - def should_have_received_frame(self, expected_frame): - self.should_have_received_frames([expected_frame], any_order=True) - - def should_have_received_method(self, channel_number, method): - self.should_have_received_methods(channel_number, [method], any_order=True) - - def should_not_have_received_method(self, channel_number, method): - results = (read(x) for x in self.data) - frames = [x for x in results if x is not None] - - frame = asynqp.frames.MethodFrame(channel_number, method) - assert frame not in frames, "{} should not have been in {}".format(frame, frames) - - def should_not_have_received_any(self): - assert not self.data, "{} should have been empty".format(self.data) - - def should_have_received_bytes(self, b): - assert b in self.data - - -def read(data): - if data == b'AMQP\x00\x00\x09\x01': - return - - result = protocol.FrameReader().read_frame(data) - if result is None: - return - return result[0] - - -def windows(l, size): - return zip(*[l[x:] for x in range(size)]) - - -class FakeTransport(object): - def __init__(self, server): - self.server = server - self.closed = False - - def write(self, data): - self.server.data.append(data) - - def close(self): - self.closed = True - - -def any(cls): - class _any(cls): - def __init__(self): - pass - - def __eq__(self, other): - return isinstance(other, cls) - return _any() - - -from contextlib import contextmanager -@contextmanager -def silence_expected_destroy_pending_log(expected_coro_name=''): - real_async = asyncio.async - def async(*args, **kwargs): - t = real_async(*args, **kwargs) - if expected_coro_name in repr(t): - t._log_destroy_pending = False - return t - - with mock.patch.object(asyncio, 'async', async): - yield +import asyncio +from contextlib import contextmanager +from unittest import mock +import asynqp.frames +from asynqp import protocol +from asynqp.exceptions import ConnectionLostError + + +def testing_exception_handler(loop, context): + ''' + Hides the expected ``ConnectionClosedErrors`` and + ``ConnectionLostErros`` during tests + ''' + exception = context.get('exception') + if exception and isinstance(exception, ConnectionLostError): + pass + else: + loop.default_exception_handler(context) + + +class MockServer(object): + def __init__(self, protocol, tick): + self.protocol = protocol + self.tick = tick + self.data = [] + + def send_bytes(self, b): + self.protocol.data_received(b) + self.tick() + + def send_frame(self, frame): + self.send_bytes(frame.serialise()) + + def send_method(self, channel_number, method): + frame = asynqp.frames.MethodFrame(channel_number, method) + self.send_frame(frame) + + def reset(self): + self.data = [] + + def should_have_received_frames(self, expected_frames, any_order=False): + results = (read(x) for x in self.data) + frames = [x for x in results if x is not None] + if any_order: + for frame in expected_frames: + assert frame in frames, "{} should have been in {}".format(frame, frames) + else: + expected_frames = tuple(expected_frames) + assert expected_frames in windows(frames, len(expected_frames)), "{} should have been in {}".format(expected_frames, frames) + + def should_have_received_methods(self, channel_number, methods, any_order=False): + frames = (asynqp.frames.MethodFrame(channel_number, m) for m in methods) + self.should_have_received_frames(frames, any_order) + + def should_have_received_frame(self, expected_frame): + self.should_have_received_frames([expected_frame], any_order=True) + + def should_have_received_method(self, channel_number, method): + self.should_have_received_methods(channel_number, [method], any_order=True) + + def should_not_have_received_method(self, channel_number, method): + results = (read(x) for x in self.data) + frames = [x for x in results if x is not None] + + frame = asynqp.frames.MethodFrame(channel_number, method) + assert frame not in frames, "{} should not have been in {}".format(frame, frames) + + def should_not_have_received_any(self): + assert not self.data, "{} should have been empty".format(self.data) + + def should_have_received_bytes(self, b): + assert b in self.data + + +def read(data): + if data == b'AMQP\x00\x00\x09\x01': + return + + result = protocol.FrameReader().read_frame(data) + if result is None: + return + return result[0] + + +def windows(l, size): + return zip(*[l[x:] for x in range(size)]) + + +class FakeTransport(object): + def __init__(self, server): + self.server = server + self.closed = False + + def write(self, data): + self.server.data.append(data) + + def close(self): + self.closed = True + + +def any(cls): + class _any(cls): + def __init__(self): + pass + + def __eq__(self, other): + return isinstance(other, cls) + return _any() + + +@contextmanager +def silence_expected_destroy_pending_log(expected_coro_name=''): + real_async = asyncio.ensure_future + + def async_wrapper(*args, **kwargs): + t = real_async(*args, **kwargs) + if expected_coro_name in repr(t): + t._log_destroy_pending = False + return t + + with mock.patch.object(asyncio, 'ensure_future', async_wrapper): + yield + + +def run_briefly(loop): + @asyncio.coroutine + def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close()