From e485b8b35d1e08c970f4a759bc270ad43530a208 Mon Sep 17 00:00:00 2001 From: Josh Watts Date: Fri, 7 Aug 2015 19:28:14 -0400 Subject: [PATCH 1/3] Allow sending request content to custom QIODevice Subclasses of HttpHandler may implement handleContent() to analyze the headers during the transition between request headers and request content. Invoking setContentDevice(contentDevice) will continue the connection, where a NULL contentDevice will behave like originally intended (direct to memory), otherwise a valid QIODevice will be written to instead. --- pillowcore/HttpConnection.cpp | 197 ++++++++++++++++++++++++++-------- pillowcore/HttpConnection.h | 7 +- pillowcore/HttpHandler.cpp | 19 ++++ pillowcore/HttpHandler.h | 2 + pillowcore/HttpServer.cpp | 17 +++ pillowcore/HttpServer.h | 1 + 6 files changed, 200 insertions(+), 43 deletions(-) diff --git a/pillowcore/HttpConnection.cpp b/pillowcore/HttpConnection.cpp index 274c78a..025b9a9 100644 --- a/pillowcore/HttpConnection.cpp +++ b/pillowcore/HttpConnection.cpp @@ -79,6 +79,7 @@ namespace Pillow public: Pillow::HttpConnection::State _state; QIODevice* _inputDevice,* _outputDevice; + QIODevice* _contentDevice; http_parser _parser; // Request fields. @@ -91,6 +92,7 @@ namespace Pillow QVarLengthArray _requestHeadersRef; Pillow::HttpHeaderCollection _requestHeaders; int _requestContentLength; int _requestContentLengthHeaderIndex; + int _requestContentReceived, _requestContentWritten; bool _requestHttp11; Pillow::HttpParamCollection _requestParams; @@ -105,7 +107,9 @@ namespace Pillow void initialize(); void processInput(); void setupRequestHeaders(); + void prepareRequestHeaders(); void transitionToReceivingHeaders(); + void setContentDevice(QIODevice *contentDevice); void transitionToReceivingContent(); void transitionToSendingHeaders(); void transitionToSendingContent(); @@ -129,7 +133,7 @@ namespace Pillow } Pillow::HttpConnectionPrivate::HttpConnectionPrivate(HttpConnection *connection) - : q_ptr(connection), _state(Pillow::HttpConnection::Uninitialized), _inputDevice(0), _outputDevice(0) + : q_ptr(connection), _state(Pillow::HttpConnection::Uninitialized), _inputDevice(0), _outputDevice(0), _contentDevice(0) { } @@ -156,8 +160,58 @@ inline void Pillow::HttpConnectionPrivate::processInput() if (_state != Pillow::HttpConnection::ReceivingHeaders && _state != Pillow::HttpConnection::ReceivingContent) return; qint64 bytesAvailable = _inputDevice->bytesAvailable(); - if (bytesAvailable > 0) + + if (_state == Pillow::HttpConnection::ReceivingContent && _contentDevice) { + // Read some data if possible + + // Limit capacity to min of MaxRequestData or bytesavailable + bytesAvailable = qMin(bytesAvailable, qint64(Pillow::HttpConnection::MaximumRequestContentLength)); + if (_requestContent.capacity() < bytesAvailable) { + qDebug("RESERVED %d -> %lld", + _requestContent.capacity(), + bytesAvailable + ); + _requestContent.reserve(bytesAvailable); + } + // Limit bytesAvailable to remaining capacity + bytesAvailable = qMin(bytesAvailable, qint64(_requestContent.capacity() - _requestContent.size())); + if (bytesAvailable > 0) { + qint64 bytesRead = _inputDevice->read(_requestContent.data() + _requestContent.size(), bytesAvailable); + _requestContent.data_ptr()->size += bytesRead; + _requestContent.data_ptr()->data[_requestContent.data_ptr()->size] = 0; + _requestContentReceived += bytesRead; + } + + // Write some data if possible + bytesAvailable = _requestContent.size(); + if (bytesAvailable > 0) { + qint64 bytesWritten = _contentDevice->write(_requestContent.data(), bytesAvailable); + if (bytesWritten < 0) + return; //TODO: Throw error, close connection + else if (bytesWritten < bytesAvailable) + { + qDebug("SHRANK %d -> %lld", + _requestContent.capacity(), + _requestContent.capacity() - bytesWritten + ); + _requestContent.remove(0, bytesWritten); + } + else + { + _requestContent.data_ptr()->size = 0; + _requestContent.data_ptr()->data[0] = 0; + } + if (_requestHeaders.size() > 0) _requestHeaders.pop_back(); + _requestContentWritten += bytesWritten; + qDebug("WROTE %d/%d", + _requestContentWritten, + _requestContentLength + ); + } + } + else if (bytesAvailable > 0) { + // Simple content or headers if (_requestBuffer.capacity() < _requestBuffer.size() + bytesAvailable) _requestBuffer.reserve(_requestBuffer.size() + bytesAvailable + 1); qint64 bytesRead = _inputDevice->read(_requestBuffer.data() + _requestBuffer.size(), bytesAvailable); @@ -174,11 +228,27 @@ inline void Pillow::HttpConnectionPrivate::processInput() return writeRequestErrorResponse(400); // Bad client Request! else if (thin_http_parser_is_finished(&_parser)) transitionToReceivingContent(); + else if (_inputDevice->bytesAvailable() > 0) + QTimer::singleShot(0, q_ptr, SLOT(processInput())); } else if (_state == Pillow::HttpConnection::ReceivingContent) { - if (_requestBuffer.size() - int(_parser.body_start) >= _requestContentLength) - transitionToSendingHeaders(); // Finished receiving the content. + if (_contentDevice) + { + if (_requestContentWritten >= _requestContentLength) + { + transitionToSendingHeaders(); + } + else if (_inputDevice->bytesAvailable() > 0 || _requestContentReceived < _requestContentLength) + QTimer::singleShot(0, q_ptr, SLOT(processInput())); + } + else + { + if (_requestContentReceived >= _requestContentLength || int(_requestBuffer.size() - _parser.body_start) >= _requestContentLength) + transitionToSendingHeaders(); // Finished receiving the content. + else if (_inputDevice->bytesAvailable() > 0 || _requestContentReceived < _requestContentLength) + QTimer::singleShot(0, q_ptr, SLOT(processInput())); + } } } @@ -193,6 +263,18 @@ inline void Pillow::HttpConnectionPrivate::transitionToReceivingHeaders() _requestHttp11 = false; } +void Pillow::HttpConnectionPrivate::setContentDevice(QIODevice *contentDevice) +{ + if (_state != Pillow::HttpConnection::ContentReady) return; + _state = Pillow::HttpConnection::ReceivingContent; + _contentDevice = contentDevice; + if (_contentDevice) { + setFromRawData(_requestContent, _requestBuffer.constData(), _parser.body_start, _requestBuffer.size() - _parser.body_start); + } + processInput(); +} + + inline void Pillow::HttpConnectionPrivate::setupRequestHeaders() { char* data = _requestBuffer.data(); @@ -209,10 +291,45 @@ inline void Pillow::HttpConnectionPrivate::setupRequestHeaders() } } +void Pillow::HttpConnectionPrivate::prepareRequestHeaders() +{ + // Prepare and null terminate the request fields. + + if (_requestHeaders.size() != _requestHeadersRef.size()) + setupRequestHeaders(); + + char* data = _requestBuffer.data(); + + if (_parser.query_string_len == 0) + { + // The request uri has no query string, we can use the data straight because it means that inserting null after the path + // will not destroy the uri (which does not include the fragment). + setFromRawDataAndNullterm(_requestUri, data, _parser.request_uri_start, _parser.request_uri_len); + } + else + { + // Create deep copy for requestUri; it would otherwise get a null inserted after the path. + _requestUri = QByteArray(data + _parser.request_uri_start, _parser.request_uri_len); + } + setFromRawDataAndNullterm(_requestMethod, data, _parser.request_method_start, _parser.request_method_len); + setFromRawDataAndNullterm(_requestFragment, data, _parser.fragment_start, _parser.fragment_len); + setFromRawDataAndNullterm(_requestPath, data, _parser.request_path_start, _parser.request_path_len); + setFromRawDataAndNullterm(_requestQueryString, data, _parser.query_string_start, _parser.query_string_len); + setFromRawDataAndNullterm(_requestHttpVersion, data, _parser.http_version_start, _parser.http_version_len); + + if (!_requestUriDecoded.isEmpty()) _requestUriDecoded = QString(); + if (!_requestFragmentDecoded.isEmpty())_requestFragmentDecoded = QString(); + if (!_requestPathDecoded.isEmpty())_requestPathDecoded = QString(); + if (!_requestQueryStringDecoded.isEmpty())_requestQueryStringDecoded = QString(); + + _requestHttp11 = _requestHttpVersion == httpSlash11Token; +} + inline void Pillow::HttpConnectionPrivate::transitionToReceivingContent() { - if (_state == Pillow::HttpConnection::ReceivingContent) return; - _state = Pillow::HttpConnection::ReceivingContent; + if (_state == Pillow::HttpConnection::ReceivingContent || + _state == Pillow::HttpConnection::ContentReady) return; + _state = Pillow::HttpConnection::ContentReady; setupRequestHeaders(); @@ -223,7 +340,7 @@ inline void Pillow::HttpConnectionPrivate::transitionToReceivingContent() // Exit early if the client sent an incorrect or unacceptable content-length. if (_requestContentLength < 0) return writeRequestErrorResponse(400); // Invalid request: negative content length does not make sense. - else if (_requestContentLength > Pillow::HttpConnection::MaximumRequestContentLength || !contentLengthParseOk) + else if (!contentLengthParseOk) return writeRequestErrorResponse(413); // Request entity too large. if (_requestContentLength > 0) @@ -234,13 +351,17 @@ inline void Pillow::HttpConnectionPrivate::transitionToReceivingContent() // Resize the request buffer right away to avoid too many reallocs later. // NOTE: This invalidates the request headers QByteArrays if the reallocation // changes the buffer's address (very likely unless the content-length is tiny). - _requestBuffer.reserve(_parser.body_start + _requestContentLength + 1); + _requestBuffer.reserve(_parser.body_start + qMin(_requestContentLength, int(Pillow::HttpConnection::MaximumRequestContentLength)) + 1); + + _requestContentReceived = _requestBuffer.size() - _parser.body_start; + _requestContentWritten = 0; // So do invalidate the request headers. if (_requestHeaders.size() > 0) _requestHeaders.pop_back(); - // Pump; the content may already be sitting in the buffers. - processInput(); + prepareRequestHeaders(); + + emit q_ptr->contentReady(q_ptr); } else { @@ -253,38 +374,10 @@ inline void Pillow::HttpConnectionPrivate::transitionToSendingHeaders() if (_state == Pillow::HttpConnection::SendingHeaders) return; _state = Pillow::HttpConnection::SendingHeaders; - // Prepare and null terminate the request fields. - - if (_requestHeaders.size() != _requestHeadersRef.size()) - setupRequestHeaders(); + prepareRequestHeaders(); - char* data = _requestBuffer.data(); - - if (_parser.query_string_len == 0) - { - // The request uri has no query string, we can use the data straight because it means that inserting null after the path - // will not destroy the uri (which does not include the fragment). - setFromRawDataAndNullterm(_requestUri, data, _parser.request_uri_start, _parser.request_uri_len); - } - else - { - // Create deep copy for requestUri; it would otherwise get a null inserted after the path. - _requestUri = QByteArray(data + _parser.request_uri_start, _parser.request_uri_len); - } - setFromRawDataAndNullterm(_requestMethod, data, _parser.request_method_start, _parser.request_method_len); - setFromRawDataAndNullterm(_requestFragment, data, _parser.fragment_start, _parser.fragment_len); - setFromRawDataAndNullterm(_requestPath, data, _parser.request_path_start, _parser.request_path_len); - setFromRawDataAndNullterm(_requestQueryString, data, _parser.query_string_start, _parser.query_string_len); - setFromRawDataAndNullterm(_requestHttpVersion, data, _parser.http_version_start, _parser.http_version_len); - - if (!_requestUriDecoded.isEmpty()) _requestUriDecoded = QString(); - if (!_requestFragmentDecoded.isEmpty())_requestFragmentDecoded = QString(); - if (!_requestPathDecoded.isEmpty())_requestPathDecoded = QString(); - if (!_requestQueryStringDecoded.isEmpty())_requestQueryStringDecoded = QString(); - - _requestHttp11 = _requestHttpVersion == httpSlash11Token; - - setFromRawData(_requestContent, _requestBuffer.constData(), _parser.body_start, _requestContentLength); + if (!_contentDevice) + setFromRawData(_requestContent, _requestBuffer.constData(), _parser.body_start, _requestContentLength); // Reset our known information about the response. _responseContentLength = -1; // The response content-length is initially unknown. @@ -651,6 +744,11 @@ void Pillow::HttpConnection::initialize(QIODevice* inputDevice, QIODevice* outpu d_ptr->initialize(); } +void Pillow::HttpConnection::setContentDevice(QIODevice *contentDevice) +{ + d_ptr->setContentDevice(contentDevice); +} + Pillow::HttpConnection::State Pillow::HttpConnection::state() const { return d_ptr->_state; @@ -666,6 +764,11 @@ QIODevice *Pillow::HttpConnection::outputDevice() const return d_ptr->_outputDevice; } +QIODevice *Pillow::HttpConnection::contentDevice() const +{ + return d_ptr->_contentDevice; +} + void Pillow::HttpConnection::processInput() { d_ptr->processInput(); @@ -693,7 +796,12 @@ void Pillow::HttpConnection::writeResponseString(int statusCode, const HttpHeade void Pillow::HttpConnection::writeHeaders(int statusCode, const HttpHeaderCollection& headers) { - d_ptr->writeHeaders(statusCode, headers); + d_ptr->writeHeaders(statusCode, headers); +} + +void Pillow::HttpConnection::writeRequestErrorResponse(int statusCode) +{ + d_ptr->writeRequestErrorResponse(statusCode); } void Pillow::HttpConnection::writeContent(const QByteArray& content) @@ -806,6 +914,11 @@ const QByteArray &Pillow::HttpConnection::requestContent() const return d_ptr->_requestContent; } +qint64 Pillow::HttpConnection::requestContentLength() const +{ + return d_ptr->_requestContentLength; +} + const Pillow::HttpHeaderCollection &Pillow::HttpConnection::requestHeaders() const { return d_ptr->_requestHeaders; diff --git a/pillowcore/HttpConnection.h b/pillowcore/HttpConnection.h index 1adb094..defbc62 100644 --- a/pillowcore/HttpConnection.h +++ b/pillowcore/HttpConnection.h @@ -42,7 +42,7 @@ namespace Pillow Q_PROPERTY(QByteArray requestContent READ requestContent NOTIFY requestReady) public: - enum State { Uninitialized, ReceivingHeaders, ReceivingContent, SendingHeaders, SendingContent, Completed, Flushing, Closed }; + enum State { Uninitialized, ReceivingHeaders, ContentReady, ReceivingContent, SendingHeaders, SendingContent, Completed, Flushing, Closed }; enum { MaximumRequestHeaderLength = 32 * 1024 }; enum { MaximumRequestContentLength = 128 * 1024 * 1024 }; Q_ENUMS(State); @@ -52,9 +52,11 @@ namespace Pillow ~HttpConnection(); void initialize(QIODevice* inputDevice, QIODevice* outputDevice = 0); + void setContentDevice(QIODevice *contentDevice); QIODevice* inputDevice() const; QIODevice* outputDevice() const; + QIODevice* contentDevice() const; State state() const; QHostAddress remoteAddress() const; @@ -68,6 +70,7 @@ namespace Pillow const QByteArray& requestQueryString() const; const QByteArray& requestHttpVersion() const; const QByteArray& requestContent() const; + qint64 requestContentLength() const; // Request members, decoded version. Use those rather than manually decoding the raw data returned by the methods // above when decoded values are desired (they are cached). @@ -91,6 +94,7 @@ namespace Pillow void writeResponse(int statusCode = 200, const Pillow::HttpHeaderCollection& headers = Pillow::HttpHeaderCollection(), const QByteArray& content = QByteArray()); void writeResponseString(int statusCode = 200, const Pillow::HttpHeaderCollection& headers = Pillow::HttpHeaderCollection(), const QString& content = QString()); void writeHeaders(int statusCode = 200, const Pillow::HttpHeaderCollection& headers = Pillow::HttpHeaderCollection()); + void writeRequestErrorResponse(int statusCode = 400); void writeContent(const QByteArray& content); void endContent(); @@ -103,6 +107,7 @@ namespace Pillow qint64 responseContentLength() const; signals: + void contentReady(Pillow::HttpConnection* self); // The headers are ready to be processed, all request headers have been received. If a contentHandler has been specified, it MUST call setContentDevice (with or without NULL) to continue handling the connection. void requestReady(Pillow::HttpConnection* self); // The request is ready to be processed, all request headers and content have been received. void requestCompleted(Pillow::HttpConnection* self); // The response is completed, all response headers and content have been sent. void closed(Pillow::HttpConnection* self); // The connection is closing, no further requests will arrive on this object. diff --git a/pillowcore/HttpHandler.cpp b/pillowcore/HttpHandler.cpp index 8eae4ff..db95f3f 100644 --- a/pillowcore/HttpHandler.cpp +++ b/pillowcore/HttpHandler.cpp @@ -19,6 +19,12 @@ HttpHandler::HttpHandler(QObject *parent) { } +bool HttpHandler::handleContent(HttpConnection *connection) +{ + connection->setContentDevice(NULL); + return false; +} + // // HttpHandlerStack // @@ -28,6 +34,19 @@ HttpHandlerStack::HttpHandlerStack(QObject *parent) { } +bool HttpHandlerStack::handleContent(HttpConnection *connection) +{ + foreach (QObject* object, children()) + { + HttpHandler* handler = qobject_cast(object); + + if (handler && handler->handleContent(connection)) + return true; + } + + return false; +} + bool HttpHandlerStack::handleRequest(Pillow::HttpConnection *connection) { foreach (QObject* object, children()) diff --git a/pillowcore/HttpHandler.h b/pillowcore/HttpHandler.h index 5f26961..2076b04 100644 --- a/pillowcore/HttpHandler.h +++ b/pillowcore/HttpHandler.h @@ -36,6 +36,7 @@ namespace Pillow HttpHandler(QObject *parent = 0); public slots: + virtual bool handleContent(Pillow::HttpConnection* connection); virtual bool handleRequest(Pillow::HttpConnection* connection) = 0; }; @@ -51,6 +52,7 @@ namespace Pillow HttpHandlerStack(QObject* parent = 0); public: + virtual bool handleContent(Pillow::HttpConnection* connection); virtual bool handleRequest(Pillow::HttpConnection *connection); }; diff --git a/pillowcore/HttpServer.cpp b/pillowcore/HttpServer.cpp index 25b7707..14dae98 100644 --- a/pillowcore/HttpServer.cpp +++ b/pillowcore/HttpServer.cpp @@ -2,6 +2,7 @@ #include "HttpConnection.h" #include #include +#include using namespace Pillow; // @@ -36,6 +37,7 @@ namespace Pillow HttpConnection* createConnection() { HttpConnection* connection = new HttpConnection(q_ptr); + QObject::connect(connection, SIGNAL(contentReady(Pillow::HttpConnection*)), q_ptr, SLOT(contentReady(Pillow::HttpConnection*))); QObject::connect(connection, SIGNAL(requestReady(Pillow::HttpConnection*)), q_ptr, SIGNAL(requestReady(Pillow::HttpConnection*))); QObject::connect(connection, SIGNAL(closed(Pillow::HttpConnection*)), q_ptr, SLOT(connection_closed(Pillow::HttpConnection*))); return connection; @@ -94,6 +96,21 @@ void HttpServer::incomingConnection(int socketDescriptor) } } +void HttpServer::contentReady(HttpConnection *connection) +{ + foreach (QObject* object, children()) + { + int methodIndex = object->metaObject()->indexOfMethod("handleContent(Pillow::HttpConnection*)"); + if (methodIndex < 0) + continue; + QMetaMethod method = object->metaObject()->method(methodIndex); + if (method.invoke(object, Q_ARG(Pillow::HttpConnection *, connection))) + return; + } + + connection->setContentDevice(NULL); +} + void HttpServer::connection_closed(Pillow::HttpConnection *connection) { connection->inputDevice()->deleteLater(); diff --git a/pillowcore/HttpServer.h b/pillowcore/HttpServer.h index 89aa56c..878c290 100644 --- a/pillowcore/HttpServer.h +++ b/pillowcore/HttpServer.h @@ -31,6 +31,7 @@ namespace Pillow HttpServerPrivate* d_ptr; private slots: + void contentReady(Pillow::HttpConnection* connection); void connection_closed(Pillow::HttpConnection* request); protected: From ff2ab662506c5477fbc218e8eb632d76ed8a8d10 Mon Sep 17 00:00:00 2001 From: Josh Watts Date: Mon, 10 Aug 2015 09:26:05 -0400 Subject: [PATCH 2/3] Oops, shouldn't call setContentDevice() if returning false... --- pillowcore/HttpHandler.cpp | 1 - pillowcore/HttpServer.cpp | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pillowcore/HttpHandler.cpp b/pillowcore/HttpHandler.cpp index db95f3f..0a074c3 100644 --- a/pillowcore/HttpHandler.cpp +++ b/pillowcore/HttpHandler.cpp @@ -21,7 +21,6 @@ HttpHandler::HttpHandler(QObject *parent) bool HttpHandler::handleContent(HttpConnection *connection) { - connection->setContentDevice(NULL); return false; } diff --git a/pillowcore/HttpServer.cpp b/pillowcore/HttpServer.cpp index 14dae98..82bdb1f 100644 --- a/pillowcore/HttpServer.cpp +++ b/pillowcore/HttpServer.cpp @@ -104,8 +104,11 @@ void HttpServer::contentReady(HttpConnection *connection) if (methodIndex < 0) continue; QMetaMethod method = object->metaObject()->method(methodIndex); - if (method.invoke(object, Q_ARG(Pillow::HttpConnection *, connection))) - return; + bool result = false; + if (method.invoke(object, Q_RETURN_ARG(bool, result), Q_ARG(Pillow::HttpConnection *, connection))) { + if (result) + return; + } } connection->setContentDevice(NULL); From 4e5d63911e2f1bd09b2c094c61cdb6fb2c76f27d Mon Sep 17 00:00:00 2001 From: Josh Watts Date: Mon, 10 Aug 2015 09:53:57 -0400 Subject: [PATCH 3/3] HttpConnection: Remove debug output --- pillowcore/HttpConnection.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pillowcore/HttpConnection.cpp b/pillowcore/HttpConnection.cpp index 025b9a9..79097cd 100644 --- a/pillowcore/HttpConnection.cpp +++ b/pillowcore/HttpConnection.cpp @@ -166,13 +166,9 @@ inline void Pillow::HttpConnectionPrivate::processInput() // Limit capacity to min of MaxRequestData or bytesavailable bytesAvailable = qMin(bytesAvailable, qint64(Pillow::HttpConnection::MaximumRequestContentLength)); - if (_requestContent.capacity() < bytesAvailable) { - qDebug("RESERVED %d -> %lld", - _requestContent.capacity(), - bytesAvailable - ); + if (_requestContent.capacity() < bytesAvailable) _requestContent.reserve(bytesAvailable); - } + // Limit bytesAvailable to remaining capacity bytesAvailable = qMin(bytesAvailable, qint64(_requestContent.capacity() - _requestContent.size())); if (bytesAvailable > 0) { @@ -190,10 +186,6 @@ inline void Pillow::HttpConnectionPrivate::processInput() return; //TODO: Throw error, close connection else if (bytesWritten < bytesAvailable) { - qDebug("SHRANK %d -> %lld", - _requestContent.capacity(), - _requestContent.capacity() - bytesWritten - ); _requestContent.remove(0, bytesWritten); } else @@ -203,10 +195,6 @@ inline void Pillow::HttpConnectionPrivate::processInput() } if (_requestHeaders.size() > 0) _requestHeaders.pop_back(); _requestContentWritten += bytesWritten; - qDebug("WROTE %d/%d", - _requestContentWritten, - _requestContentLength - ); } } else if (bytesAvailable > 0)