From 499aa3d748dada60edd32a1406fb38796b7db441 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Tue, 9 Oct 2018 13:26:43 -0400 Subject: [PATCH 01/17] Initial check-in of SQS work --- composer.json | 3 +- composer.lock | 370 +++++++++++++++++- includes/WpMinions/Plugin.php | 4 + .../WpMinions/SimpleQueueService/Client.php | 149 +++++++ .../SimpleQueueService/Connection.php | 7 + .../WpMinions/SimpleQueueService/Worker.php | 137 +++++++ readme.md | 2 +- 7 files changed, 668 insertions(+), 4 deletions(-) create mode 100644 includes/WpMinions/SimpleQueueService/Client.php create mode 100644 includes/WpMinions/SimpleQueueService/Connection.php create mode 100644 includes/WpMinions/SimpleQueueService/Worker.php diff --git a/composer.json b/composer.json index 1df355c..225ad84 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,8 @@ ], "require": { "composer/installers": "~1.0", - "php-amqplib/php-amqplib": ">=2.6.1" + "php-amqplib/php-amqplib": ">=2.6.1", + "aws/aws-sdk-php": "^3.64" }, "require-dev": { "phpunit/phpunit": "~3.7", diff --git a/composer.lock b/composer.lock index f710ec0..3131a95 100644 --- a/composer.lock +++ b/composer.lock @@ -1,11 +1,91 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3da539add4bc42dcecdaa80e0c670adc", + "content-hash": "d3dcdd83f1248fda68e1549370859fd1", "packages": [ + { + "name": "aws/aws-sdk-php", + "version": "3.64.4", + "source": { + "type": "git", + "url": "https://github.com/aws/aws-sdk-php.git", + "reference": "f053be09055d43bd60aefd9998440479a45c3da9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f053be09055d43bd60aefd9998440479a45c3da9", + "reference": "f053be09055d43bd60aefd9998440479a45c3da9", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-pcre": "*", + "ext-simplexml": "*", + "ext-spl": "*", + "guzzlehttp/guzzle": "^5.3.1|^6.2.1", + "guzzlehttp/promises": "~1.0", + "guzzlehttp/psr7": "^1.4.1", + "mtdowling/jmespath.php": "~2.2", + "php": ">=5.5" + }, + "require-dev": { + "andrewsville/php-token-reflection": "^1.4", + "aws/aws-php-sns-message-validator": "~1.0", + "behat/behat": "~3.0", + "doctrine/cache": "~1.4", + "ext-dom": "*", + "ext-openssl": "*", + "nette/neon": "^2.3", + "phpunit/phpunit": "^4.8.35|^5.4.3", + "psr/cache": "^1.0" + }, + "suggest": { + "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", + "doctrine/cache": "To use the DoctrineCacheAdapter", + "ext-curl": "To send requests using cURL", + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "psr-4": { + "Aws\\": "src/" + }, + "files": [ + "src/functions.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Amazon Web Services", + "homepage": "http://aws.amazon.com" + } + ], + "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", + "homepage": "http://aws.amazon.com/sdkforphp", + "keywords": [ + "amazon", + "aws", + "cloud", + "dynamodb", + "ec2", + "glacier", + "s3", + "sdk" + ], + "time": "2018-08-02T20:23:36+00:00" + }, { "name": "composer/installers", "version": "v1.3.0", @@ -120,6 +200,242 @@ ], "time": "2017-04-24T06:37:16+00:00" }, + { + "name": "guzzlehttp/guzzle", + "version": "6.3.3", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle.git", + "reference": "407b0cb880ace85c9b63c5f9551db498cb2d50ba" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/407b0cb880ace85c9b63c5f9551db498cb2d50ba", + "reference": "407b0cb880ace85c9b63c5f9551db498cb2d50ba", + "shasum": "" + }, + "require": { + "guzzlehttp/promises": "^1.0", + "guzzlehttp/psr7": "^1.4", + "php": ">=5.5" + }, + "require-dev": { + "ext-curl": "*", + "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.4 || ^7.0", + "psr/log": "^1.0" + }, + "suggest": { + "psr/log": "Required for using the Log middleware" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "6.3-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "GuzzleHttp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Guzzle is a PHP HTTP client library", + "homepage": "http://guzzlephp.org/", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "rest", + "web service" + ], + "time": "2018-04-22T15:46:56+00:00" + }, + { + "name": "guzzlehttp/promises", + "version": "v1.3.1", + "source": { + "type": "git", + "url": "https://github.com/guzzle/promises.git", + "reference": "a59da6cf61d80060647ff4d3eb2c03a2bc694646" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/promises/zipball/a59da6cf61d80060647ff4d3eb2c03a2bc694646", + "reference": "a59da6cf61d80060647ff4d3eb2c03a2bc694646", + "shasum": "" + }, + "require": { + "php": ">=5.5.0" + }, + "require-dev": { + "phpunit/phpunit": "^4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.4-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Promise\\": "src/" + }, + "files": [ + "src/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Guzzle promises library", + "keywords": [ + "promise" + ], + "time": "2016-12-20T10:07:11+00:00" + }, + { + "name": "guzzlehttp/psr7", + "version": "1.4.2", + "source": { + "type": "git", + "url": "https://github.com/guzzle/psr7.git", + "reference": "f5b8a8512e2b58b0071a7280e39f14f72e05d87c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/f5b8a8512e2b58b0071a7280e39f14f72e05d87c", + "reference": "f5b8a8512e2b58b0071a7280e39f14f72e05d87c", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "psr/http-message": "~1.0" + }, + "provide": { + "psr/http-message-implementation": "1.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.4-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Psr7\\": "src/" + }, + "files": [ + "src/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Tobias Schultze", + "homepage": "https://github.com/Tobion" + } + ], + "description": "PSR-7 message implementation that also provides common utility methods", + "keywords": [ + "http", + "message", + "request", + "response", + "stream", + "uri", + "url" + ], + "time": "2017-03-20T17:10:46+00:00" + }, + { + "name": "mtdowling/jmespath.php", + "version": "2.4.0", + "source": { + "type": "git", + "url": "https://github.com/jmespath/jmespath.php.git", + "reference": "adcc9531682cf87dfda21e1fd5d0e7a41d292fac" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/adcc9531682cf87dfda21e1fd5d0e7a41d292fac", + "reference": "adcc9531682cf87dfda21e1fd5d0e7a41d292fac", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.0" + }, + "bin": [ + "bin/jp.php" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "psr-4": { + "JmesPath\\": "src/" + }, + "files": [ + "src/JmesPath.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + } + ], + "description": "Declaratively specify how to extract elements from a JSON document", + "keywords": [ + "json", + "jsonpath" + ], + "time": "2016-12-03T22:08:25+00:00" + }, { "name": "php-amqplib/php-amqplib", "version": "v2.6.3", @@ -189,6 +505,56 @@ "rabbitmq" ], "time": "2016-04-11T14:30:01+00:00" + }, + { + "name": "psr/http-message", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-message.git", + "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363", + "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP messages", + "homepage": "https://github.com/php-fig/http-message", + "keywords": [ + "http", + "http-message", + "psr", + "psr-7", + "request", + "response" + ], + "time": "2016-08-06T14:39:51+00:00" } ], "packages-dev": [ diff --git a/includes/WpMinions/Plugin.php b/includes/WpMinions/Plugin.php index f0c25bd..4cc166e 100644 --- a/includes/WpMinions/Plugin.php +++ b/includes/WpMinions/Plugin.php @@ -187,6 +187,8 @@ function build_client() { return new \WpMinions\Gearman\Client(); } elseif ( 'rabbitmq' === strtolower( $backend ) ) { return new \WpMinions\RabbitMQ\Client(); + } elseif ( 'sqs' === strtolower( $backend ) ) { + return new \WpMinions\SimpleQueueService\Client(); } else { return new \WpMinions\Cron\Client(); } @@ -217,6 +219,8 @@ function build_worker() { return new \WpMinions\Gearman\Worker(); } elseif ( 'rabbitmq' === strtolower( $backend ) ) { return new \WpMinions\RabbitMQ\Worker(); + } elseif ( 'sqs' === strtolower( $backend ) ) { + return new \WpMinions\SimpleQueueService\Worker(); } else { return new \WpMinions\Cron\Worker(); } diff --git a/includes/WpMinions/SimpleQueueService/Client.php b/includes/WpMinions/SimpleQueueService/Client.php new file mode 100644 index 0000000..ba435f7 --- /dev/null +++ b/includes/WpMinions/SimpleQueueService/Client.php @@ -0,0 +1,149 @@ +get_sqs_client(); + + if ( ! $client ) { + return false; + } + } + + /** + * Adds a Job to the SQS Queue. + * + * @param string $hook The action hook name for the job + * @param array $args Optional arguments for the job + * @param string $priority Optional priority of the job (ignored for SQS) + * @return bool true or false depending on the Client + */ + public function add( $hook, $args = array(), $priority = 'normal' ) { + $job_data = array( + 'hook' => $hook, + 'args' => $args, + 'blog_id' => $this->get_blog_id(), + ); + + $client = $this->get_sqs_client(); + + if ( $client !== false ) { + $payload = json_encode( $job_data ); + $queueURL = $client->createQueue( + array( + 'QueueName' => $this->get_queue_name() + ) + ); + $callable = array( $client, 'sendMessage' ); + + return call_user_func( $callable, array( + 'QueueUrl' => $queueURL, + 'MessageBody' => $payload, + ) ); + } else { + return false; + } + } + + /* Helpers */ + + /** + * The Function Group used to split libGearman functions on a + * multi-network install. + * + * @return string The prefixed group name + */ + function get_queue_name() { + $key = ''; + + if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { + $key .= WP_ASYNC_TASK_SALT . ':'; + } + + $key .= 'WP_Async_Task'; + + return $key; + } + + /** + * Builds the SQS Client Instance if the extension is + * installed. Once created returns the previous instance without + * reinitialization. + * + * @return Aws\Sqs\SqsClient|false An instance of SqsClient + */ + function get_sqs_client() { + if ( is_null( $this->sqs_client ) ) { + if ( class_exists( 'SqsClient' ) ) { + $this->sqs_client = SqsClient::factory(array( + 'profile' => self::get_profile_name(), + 'region' => self::get_region_name() + )); + } else { + $this->sqs_client = false; + } + } + + return $this->sqs_client; + } + + /** + * Retrieves the region for this queue. + * Looks in the AWS_DEFAULT_REGION environment variable. + * Defaults to a hard-coded value. + * @return string region name + */ + static function get_region_name() { + if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { + return $_ENV['AWS_DEFAULT_REGION ']; + } + else { + return 'us-east-1'; + } + } + + /** + * Retrieves the profile name for this queue. + * Looks in the AWS_PROFILE environment variable. + * Defaults to 'default'. + * @return string profile name + */ + static function get_profile_name() { + if( isset( $_ENV['AWS_PROFILE'] ) ) { + return $_ENV['AWS_PROFILE']; + } + else { + return 'default'; + } + } + + /** + * Caches and returns the current blog id for adding to the Job meta + * data. False if not a multisite install. + * + * @return int|false The current blog ids id. + */ + static function get_blog_id() { + return function_exists( 'is_multisite' ) && is_multisite() ? get_current_blog_id() : false; + } + +} diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php new file mode 100644 index 0000000..250f09c --- /dev/null +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -0,0 +1,7 @@ +get_sqs_client(); + + if ( ! $client ) { + return false; + } + } + + /** + * Pulls a job from the SQS Queue and tries to execute it. + * Errors are logged if the Job failed to execute. + * + * @return bool True if the job could be executed, else false + */ + public function work() { + $client = $this->get_sqs_client(); + + try { + $result = $worker->work(); + } catch ( \Exception $e ) { + if ( ! defined( 'PHPUNIT_RUNNER' ) ) { + error_log( 'SQSWorker->work failed: ' . $e->getMessage() ); + } + $result = false; + } + + do_action( 'wp_async_task_after_work', $result, $this ); + + return $result; + } + + /* Helpers */ + /** + * Executes a Job pulled from SQS. On a multisite instance + * it switches to the target site before executing the job. And the + * site is restored once executing is finished. + * + * The job data contains, + * + * 1. hook - The name of the target hook to execute + * 2. args - Optional arguments to pass to the target hook + * 3. blog_id - Optional blog on a multisite to switch to, before execution + * + * Actions are fired before and after execution of the target hook. + * + * Eg:- for the action 'foo' The order of execution of actions is, + * + * 1. wp_async_task_before_job + * 2. wp_async_task_before_job_foo + * 3. foo + * 4. wp_async_task_after_job + * 5. wp_async_task_after_job_foo + * + * @param array $job The job object data. + * @return bool True or false based on the status of execution + */ + function do_job( $job ) { + $switched = false; + + try { + $job_data = json_decode( $job->workload(), true ); + $hook = $job_data['hook']; + $args = $job_data['args']; + + if ( function_exists( 'is_multisite' ) && is_multisite() && $job_data['blog_id'] ) { + $blog_id = $job_data['blog_id']; + + if ( get_current_blog_id() !== $blog_id ) { + switch_to_blog( $blog_id ); + $switched = true; + } else { + $switched = false; + } + } else { + $switched = false; + } + + do_action( 'wp_async_task_before_job', $hook, $job ); + do_action( 'wp_async_task_before_job_' . $hook, $job ); + + do_action( $hook, $args, $job ); + + do_action( 'wp_async_task_after_job', $hook, $job ); + do_action( 'wp_async_task_after_job_' . $hook, $job ); + + $result = true; + } catch ( \Exception $e ) { + error_log( + 'SQSWorker->do_job failed: ' . $e->getMessage() + ); + $result = false; + } + + if ( $switched ) { + restore_current_blog(); + } + + return $result; + } + + /** + * The Function Group used to split libSQS functions on a + * multi-network install. + * + * @return string The prefixed group name + */ + function get_async_group() { + $key = ''; + + if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { + $key .= WP_ASYNC_TASK_SALT . ':'; + } + + $key .= 'WP_Async_Task'; + + return $key; + } + +} diff --git a/readme.md b/readme.md index 1adc41a..4f88c95 100644 --- a/readme.md +++ b/readme.md @@ -151,7 +151,7 @@ Where 'n' is the number of processes you want. #### WordPress Configuration -Define the `WP_MINIONS_BACKEND` constant in your ```wp-config.php```. Valid values are `gearman` or `rabbitmq`. If left blank, it will default to a cron client. +Define the `WP_MINIONS_BACKEND` constant in your ```wp-config.php```. Valid values are `gearman`, `rabbitmq`, or `sqs`. If left blank, it will default to a cron client. ``` define( 'WP_MINIONS_BACKEND', 'gearman' ); ``` From 7169305ba480a7c904354b106bbc2dffed261427 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 09:53:43 -0400 Subject: [PATCH 02/17] More development --- .../WpMinions/SimpleQueueService/Client.php | 31 ++-- .../WpMinions/SimpleQueueService/Worker.php | 144 ++++++++++++++++-- 2 files changed, 155 insertions(+), 20 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Client.php b/includes/WpMinions/SimpleQueueService/Client.php index ba435f7..82d561b 100644 --- a/includes/WpMinions/SimpleQueueService/Client.php +++ b/includes/WpMinions/SimpleQueueService/Client.php @@ -3,7 +3,9 @@ namespace WpMinions\SimpleQueueService; use WpMinions\Client as BaseClient; -use Aws\Sqs\SqsClient; +use Aws\Sqs\SqsClient as SqsClient; +use Aws\Exception\AwsException as AwsException; +use \RuntimeException as RuntimeException; /** */ @@ -14,7 +16,6 @@ class Client extends BaseClient { */ public $sqs_client; - /** * Creates a new SQS Client instance and configures the * queue that it should connect to. @@ -22,11 +23,15 @@ class Client extends BaseClient { * @return bool True or false if successful */ public function register() { - $client = $this->get_sqs_client(); - - if ( ! $client ) { - return false; + try { + $client = $this->get_sqs_client(); } + catch (Exception $e) { + error_log( "Fatal SQS Error: Failed to connect" ); + error_log( " Cause: " . $e->getMessage() ); + } + + return $client !== false; } /** @@ -48,17 +53,19 @@ public function add( $hook, $args = array(), $priority = 'normal' ) { if ( $client !== false ) { $payload = json_encode( $job_data ); - $queueURL = $client->createQueue( + $result = $client->createQueue( array( 'QueueName' => $this->get_queue_name() ) ); + $callable = array( $client, 'sendMessage' ); return call_user_func( $callable, array( - 'QueueUrl' => $queueURL, + 'QueueUrl' => $result['QueueUrl'], 'MessageBody' => $payload, ) ); + } else { return false; } @@ -76,7 +83,7 @@ function get_queue_name() { $key = ''; if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { - $key .= WP_ASYNC_TASK_SALT . ':'; + $key .= WP_ASYNC_TASK_SALT . '-'; } $key .= 'WP_Async_Task'; @@ -93,13 +100,15 @@ function get_queue_name() { */ function get_sqs_client() { if ( is_null( $this->sqs_client ) ) { - if ( class_exists( 'SqsClient' ) ) { + if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { $this->sqs_client = SqsClient::factory(array( + 'version' => '2012-11-05', 'profile' => self::get_profile_name(), - 'region' => self::get_region_name() + 'region' => self::get_region_name(), )); } else { $this->sqs_client = false; + throw new RuntimeException('AWS SDK not loaded'); } } diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 9a0a831..1817b24 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -4,12 +4,18 @@ use WpMinions\Worker as BaseWorker; use Aws\Sqs\SqsClient; +use Aws\Result; /** * */ class Worker extends BaseWorker { + /** + * @var SqsClient The AWS SDK for PHP Client instance + */ + public $sqs_client; + /** * Creates a SQS Worker and initializes the servers it should * connect to. The callback that will execute a job's hook is also setup here. @@ -17,11 +23,15 @@ class Worker extends BaseWorker { * @return bool True if operation was successful else false. */ public function register() { - $client = $this->get_sqs_client(); - - if ( ! $client ) { - return false; + try { + $client = $this->get_sqs_client(); } + catch (Exception $e) { + error_log( "Fatal SQS Error: Failed to connect" ); + error_log( " Cause: " . $e->getMessage() ); + } + + return $client !== false; } /** @@ -31,20 +41,64 @@ public function register() { * @return bool True if the job could be executed, else false */ public function work() { + $payload = false; + $receiptHandle = false; $client = $this->get_sqs_client(); try { - $result = $worker->work(); + if ( $client !== false ) { + $createQueueResult = $client->createQueue( + array( + 'QueueName' => $this->get_queue_name() + ) + ); + + $callable = array( $client, 'receiveMessage' ); + + $receiveMessageResult = call_user_func( $callable, array( + 'QueueUrl' => $createQueueResult['QueueUrl'], + 'MaxNumberOfMessages' => 1, + ) ); + + if( $receiveMessageResult instanceof \Aws\Result ) { + $messages = $receiveMessageResult->get('Messages'); + if( isset( $messages[0]['Body'] ) ) { + $payload = $messages[0]['Body']; + } + if( isset( $messages[0]['ReceiptHandle'] ) ) { + $receiptHandle = $messages[0]['ReceiptHandle']; + } + } + + } + } catch ( \Exception $e ) { if ( ! defined( 'PHPUNIT_RUNNER' ) ) { - error_log( 'SQSWorker->work failed: ' . $e->getMessage() ); + error_log( 'SQSWorker failed to get message: ' . $e->getMessage() ); } - $result = false; } - do_action( 'wp_async_task_after_work', $result, $this ); + do_action( 'wp_async_task_after_work', $payload, $this ); - return $result; + if( !empty( $payload ) && !empty( $receiptHandle ) ) { + + try { + $callable = array( $client, 'deleteMessage' ); + + $deleteMessageResult = call_user_func( $callable, array( + 'QueueUrl' => $createQueueResult['QueueUrl'], + 'ReceiptHandle' => $receiptHandle, + ) ); + + } + catch ( \Exception $e ) { + if ( ! defined( 'PHPUNIT_RUNNER' ) ) { + error_log( 'SQSWorker failed to delete message: ' . $e->getMessage() ); + } + } + } + + return $payload !== false; } /* Helpers */ @@ -134,4 +188,76 @@ function get_async_group() { return $key; } + /** + * Retrieves the region for this queue. + * Looks in the AWS_DEFAULT_REGION environment variable. + * Defaults to a hard-coded value. + * @return string region name + */ + static function get_region_name() { + if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { + return $_ENV['AWS_DEFAULT_REGION ']; + } + else { + return 'us-east-1'; + } + } + + /** + * Retrieves the profile name for this queue. + * Looks in the AWS_PROFILE environment variable. + * Defaults to 'default'. + * @return string profile name + */ + static function get_profile_name() { + if( isset( $_ENV['AWS_PROFILE'] ) ) { + return $_ENV['AWS_PROFILE']; + } + else { + return 'default'; + } + } + + /** + * The Function Group used to split libGearman functions on a + * multi-network install. + * + * @return string The prefixed group name + */ + function get_queue_name() { + $key = ''; + + if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { + $key .= WP_ASYNC_TASK_SALT . '-'; + } + + $key .= 'WP_Async_Task'; + + return $key; + } + + /** + * Builds the SQS Client Instance if the extension is + * installed. Once created returns the previous instance without + * reinitialization. + * + * @return Aws\Sqs\SqsClient|false An instance of SqsClient + */ + function get_sqs_client() { + if ( is_null( $this->sqs_client ) ) { + if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { + $this->sqs_client = SqsClient::factory(array( + 'version' => '2012-11-05', + 'profile' => self::get_profile_name(), + 'region' => self::get_region_name(), + )); + } else { + $this->sqs_client = false; + throw new RuntimeException('AWS SDK not loaded'); + } + } + + return $this->sqs_client; + } + } From 545f7e4a6f85050091b990fa6bc26771d65ecf68 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 17:45:16 -0400 Subject: [PATCH 03/17] Extracted duplicate methods into a Connection class --- .../WpMinions/SimpleQueueService/Client.php | 61 +----------- .../SimpleQueueService/Connection.php | 70 ++++++++++++- .../WpMinions/SimpleQueueService/Worker.php | 99 +++---------------- 3 files changed, 83 insertions(+), 147 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Client.php b/includes/WpMinions/SimpleQueueService/Client.php index 82d561b..1ca9a2d 100644 --- a/includes/WpMinions/SimpleQueueService/Client.php +++ b/includes/WpMinions/SimpleQueueService/Client.php @@ -55,7 +55,7 @@ public function add( $hook, $args = array(), $priority = 'normal' ) { $payload = json_encode( $job_data ); $result = $client->createQueue( array( - 'QueueName' => $this->get_queue_name() + 'QueueName' => Connection::get_queue_name() ) ); @@ -73,24 +73,6 @@ public function add( $hook, $args = array(), $priority = 'normal' ) { /* Helpers */ - /** - * The Function Group used to split libGearman functions on a - * multi-network install. - * - * @return string The prefixed group name - */ - function get_queue_name() { - $key = ''; - - if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { - $key .= WP_ASYNC_TASK_SALT . '-'; - } - - $key .= 'WP_Async_Task'; - - return $key; - } - /** * Builds the SQS Client Instance if the extension is * installed. Once created returns the previous instance without @@ -100,51 +82,12 @@ function get_queue_name() { */ function get_sqs_client() { if ( is_null( $this->sqs_client ) ) { - if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { - $this->sqs_client = SqsClient::factory(array( - 'version' => '2012-11-05', - 'profile' => self::get_profile_name(), - 'region' => self::get_region_name(), - )); - } else { - $this->sqs_client = false; - throw new RuntimeException('AWS SDK not loaded'); - } + $this->sqs_client = Connection::connect(); } return $this->sqs_client; } - /** - * Retrieves the region for this queue. - * Looks in the AWS_DEFAULT_REGION environment variable. - * Defaults to a hard-coded value. - * @return string region name - */ - static function get_region_name() { - if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { - return $_ENV['AWS_DEFAULT_REGION ']; - } - else { - return 'us-east-1'; - } - } - - /** - * Retrieves the profile name for this queue. - * Looks in the AWS_PROFILE environment variable. - * Defaults to 'default'. - * @return string profile name - */ - static function get_profile_name() { - if( isset( $_ENV['AWS_PROFILE'] ) ) { - return $_ENV['AWS_PROFILE']; - } - else { - return 'default'; - } - } - /** * Caches and returns the current blog id for adding to the Job meta * data. False if not a multisite install. diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php index 250f09c..99536ca 100644 --- a/includes/WpMinions/SimpleQueueService/Connection.php +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -2,6 +2,74 @@ namespace WpMinions\SimpleQueueService; +use Aws\Sqs\SqsClient; + class Connection { - + + /** + * Establish a connection to AWS SQS + * @throws RuntimeException is AWS PHP SDK isn't loaded + */ + public static function connect() { + if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { + return SqsClient::factory(array( + 'version' => '2012-11-05', + 'profile' => self::get_profile_name(), + 'region' => self::get_region_name(), + )); + } else { + throw new RuntimeException('AWS SDK not loaded'); + } + } + + /** + * The Function Group used to split libGearman functions on a + * multi-network install. + * + * @return string The prefixed group name + */ + public static function get_queue_name() { + $key = ''; + + if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { + $key .= WP_ASYNC_TASK_SALT . '-'; + } + + $key .= 'WP_Async_Task'; + + return $key; + } + + /** + * Retrieves the AWS region for this queue. + * Looks in the AWS_DEFAULT_REGION environment variable. + * Defaults to a hard-coded value. + * @param string $default default value + * @return string region name + */ + public static function get_region_name( $default = 'us-east-1' ) { + if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { + return $_ENV['AWS_DEFAULT_REGION ']; + } + else { + return $default; + } + } + + /** + * Retrieves the profile name for this queue. + * Looks in the AWS_PROFILE environment variable. + * Defaults to 'default'. + * @param string $default default value + * @return string profile name + */ + public static function get_profile_name( $default = 'default' ) { + if( isset( $_ENV['AWS_PROFILE'] ) ) { + return $_ENV['AWS_PROFILE']; + } + else { + return $default; + } + } + } \ No newline at end of file diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 1817b24..df6fd09 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -3,7 +3,6 @@ namespace WpMinions\SimpleQueueService; use WpMinions\Worker as BaseWorker; -use Aws\Sqs\SqsClient; use Aws\Result; /** @@ -17,21 +16,21 @@ class Worker extends BaseWorker { public $sqs_client; /** - * Creates a SQS Worker and initializes the servers it should - * connect to. The callback that will execute a job's hook is also setup here. + * Creates a SQS Worker and connects to AWS SQS. + * The callback that will execute a job's hook is also setup here. * * @return bool True if operation was successful else false. */ public function register() { try { - $client = $this->get_sqs_client(); + $this->sqs_client = $this->get_sqs_client(); } catch (Exception $e) { error_log( "Fatal SQS Error: Failed to connect" ); error_log( " Cause: " . $e->getMessage() ); } - return $client !== false; + return $this->sqs_client !== false; } /** @@ -43,17 +42,16 @@ public function register() { public function work() { $payload = false; $receiptHandle = false; - $client = $this->get_sqs_client(); try { - if ( $client !== false ) { - $createQueueResult = $client->createQueue( + if ( $this->sqs_client !== false ) { + $createQueueResult = $this->sqs_client->createQueue( array( - 'QueueName' => $this->get_queue_name() + 'QueueName' => Connection::get_queue_name() ) ); - $callable = array( $client, 'receiveMessage' ); + $callable = array( $this->sqs_client, 'receiveMessage' ); $receiveMessageResult = call_user_func( $callable, array( 'QueueUrl' => $createQueueResult['QueueUrl'], @@ -83,7 +81,7 @@ public function work() { if( !empty( $payload ) && !empty( $receiptHandle ) ) { try { - $callable = array( $client, 'deleteMessage' ); + $callable = array( $this->sqs_client, 'deleteMessage' ); $deleteMessageResult = call_user_func( $callable, array( 'QueueUrl' => $createQueueResult['QueueUrl'], @@ -102,6 +100,7 @@ public function work() { } /* Helpers */ + /** * Executes a Job pulled from SQS. On a multisite instance * it switches to the target site before executing the job. And the @@ -157,6 +156,7 @@ function do_job( $job ) { $result = true; } catch ( \Exception $e ) { + error_log( 'SQSWorker->do_job failed: ' . $e->getMessage() ); @@ -170,72 +170,6 @@ function do_job( $job ) { return $result; } - /** - * The Function Group used to split libSQS functions on a - * multi-network install. - * - * @return string The prefixed group name - */ - function get_async_group() { - $key = ''; - - if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { - $key .= WP_ASYNC_TASK_SALT . ':'; - } - - $key .= 'WP_Async_Task'; - - return $key; - } - - /** - * Retrieves the region for this queue. - * Looks in the AWS_DEFAULT_REGION environment variable. - * Defaults to a hard-coded value. - * @return string region name - */ - static function get_region_name() { - if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { - return $_ENV['AWS_DEFAULT_REGION ']; - } - else { - return 'us-east-1'; - } - } - - /** - * Retrieves the profile name for this queue. - * Looks in the AWS_PROFILE environment variable. - * Defaults to 'default'. - * @return string profile name - */ - static function get_profile_name() { - if( isset( $_ENV['AWS_PROFILE'] ) ) { - return $_ENV['AWS_PROFILE']; - } - else { - return 'default'; - } - } - - /** - * The Function Group used to split libGearman functions on a - * multi-network install. - * - * @return string The prefixed group name - */ - function get_queue_name() { - $key = ''; - - if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { - $key .= WP_ASYNC_TASK_SALT . '-'; - } - - $key .= 'WP_Async_Task'; - - return $key; - } - /** * Builds the SQS Client Instance if the extension is * installed. Once created returns the previous instance without @@ -245,16 +179,7 @@ function get_queue_name() { */ function get_sqs_client() { if ( is_null( $this->sqs_client ) ) { - if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { - $this->sqs_client = SqsClient::factory(array( - 'version' => '2012-11-05', - 'profile' => self::get_profile_name(), - 'region' => self::get_region_name(), - )); - } else { - $this->sqs_client = false; - throw new RuntimeException('AWS SDK not loaded'); - } + $this->sqs_client = Connection::connect(); } return $this->sqs_client; From 8fa70a6bfb8942a795223e920ca419f60097af0d Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 17:52:53 -0400 Subject: [PATCH 04/17] Updated composer, including AWS SDK and dependencies --- composer.lock | 127 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 29 deletions(-) diff --git a/composer.lock b/composer.lock index 3131a95..d1e58c4 100644 --- a/composer.lock +++ b/composer.lock @@ -8,16 +8,16 @@ "packages": [ { "name": "aws/aws-sdk-php", - "version": "3.64.4", + "version": "3.69.3", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "f053be09055d43bd60aefd9998440479a45c3da9" + "reference": "92ade997fc057d22bbee902468f749ef8db1c162" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/f053be09055d43bd60aefd9998440479a45c3da9", - "reference": "f053be09055d43bd60aefd9998440479a45c3da9", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/92ade997fc057d22bbee902468f749ef8db1c162", + "reference": "92ade997fc057d22bbee902468f749ef8db1c162", "shasum": "" }, "require": { @@ -38,6 +38,8 @@ "doctrine/cache": "~1.4", "ext-dom": "*", "ext-openssl": "*", + "ext-pcntl": "*", + "ext-sockets": "*", "nette/neon": "^2.3", "phpunit/phpunit": "^4.8.35|^5.4.3", "psr/cache": "^1.0" @@ -46,7 +48,8 @@ "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", "doctrine/cache": "To use the DoctrineCacheAdapter", "ext-curl": "To send requests using cURL", - "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages" + "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", + "ext-sockets": "To use client-side monitoring" }, "type": "library", "extra": { @@ -84,20 +87,20 @@ "s3", "sdk" ], - "time": "2018-08-02T20:23:36+00:00" + "time": "2018-10-09T20:35:16+00:00" }, { "name": "composer/installers", - "version": "v1.3.0", + "version": "v1.6.0", "source": { "type": "git", "url": "https://github.com/composer/installers.git", - "reference": "79ad876c7498c0bbfe7eed065b8651c93bfd6045" + "reference": "cfcca6b1b60bc4974324efb5783c13dca6932b5b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/installers/zipball/79ad876c7498c0bbfe7eed065b8651c93bfd6045", - "reference": "79ad876c7498c0bbfe7eed065b8651c93bfd6045", + "url": "https://api.github.com/repos/composer/installers/zipball/cfcca6b1b60bc4974324efb5783c13dca6932b5b", + "reference": "cfcca6b1b60bc4974324efb5783c13dca6932b5b", "shasum": "" }, "require": { @@ -109,7 +112,7 @@ }, "require-dev": { "composer/composer": "1.0.*@dev", - "phpunit/phpunit": "4.1.*" + "phpunit/phpunit": "^4.8.36" }, "type": "composer-plugin", "extra": { @@ -143,6 +146,7 @@ "Hurad", "ImageCMS", "Kanboard", + "Lan Management System", "MODX Evo", "Mautic", "Maya", @@ -166,6 +170,7 @@ "croogo", "dokuwiki", "drupal", + "eZ Platform", "elgg", "expressionengine", "fuelphp", @@ -178,14 +183,18 @@ "lavalite", "lithium", "magento", + "majima", "mako", "mediawiki", "modulework", + "modx", "moodle", + "osclass", "phpbb", "piwik", "ppi", "puppet", + "pxcms", "reindex", "roundcube", "shopware", @@ -198,7 +207,7 @@ "zend", "zikula" ], - "time": "2017-04-24T06:37:16+00:00" + "time": "2018-08-27T06:10:37+00:00" }, { "name": "guzzlehttp/guzzle", @@ -438,16 +447,16 @@ }, { "name": "php-amqplib/php-amqplib", - "version": "v2.6.3", + "version": "v2.7.2", "source": { "type": "git", "url": "https://github.com/php-amqplib/php-amqplib.git", - "reference": "fa2f0d4410a11008cb36b379177291be7ee9e4f6" + "reference": "dfd3694a86f1a7394d3693485259d4074a6ec79b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/fa2f0d4410a11008cb36b379177291be7ee9e4f6", - "reference": "fa2f0d4410a11008cb36b379177291be7ee9e4f6", + "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/dfd3694a86f1a7394d3693485259d4074a6ec79b", + "reference": "dfd3694a86f1a7394d3693485259d4074a6ec79b", "shasum": "" }, "require": { @@ -459,6 +468,7 @@ "videlalvaro/php-amqplib": "self.version" }, "require-dev": { + "phpdocumentor/phpdocumentor": "^2.9", "phpunit/phpunit": "^4.8", "scrutinizer/ocular": "^1.1", "squizlabs/php_codesniffer": "^2.5" @@ -479,7 +489,7 @@ }, "notification-url": "https://packagist.org/downloads/", "license": [ - "LGPL-2.1" + "LGPL-2.1-or-later" ], "authors": [ { @@ -504,7 +514,7 @@ "queue", "rabbitmq" ], - "time": "2016-04-11T14:30:01+00:00" + "time": "2018-02-11T19:28:00+00:00" }, { "name": "psr/http-message", @@ -731,16 +741,16 @@ }, { "name": "phpunit/php-file-iterator", - "version": "1.4.2", + "version": "1.4.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5" + "reference": "730b01bc3e867237eaac355e06a36b85dd93a8b4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/3cc8f69b3028d0f96a9078e6295d86e9bf019be5", - "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/730b01bc3e867237eaac355e06a36b85dd93a8b4", + "reference": "730b01bc3e867237eaac355e06a36b85dd93a8b4", "shasum": "" }, "require": { @@ -774,7 +784,7 @@ "filesystem", "iterator" ], - "time": "2016-10-03T07:40:28+00:00" + "time": "2017-11-27T13:52:08+00:00" }, { "name": "phpunit/php-text-template", @@ -1038,22 +1048,81 @@ ], "time": "2013-01-13T10:24:48+00:00" }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.9.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "e3d826245268269cd66f8326bd8bc066687b4a19" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/e3d826245268269cd66f8326bd8bc066687b4a19", + "reference": "e3d826245268269cd66f8326bd8bc066687b4a19", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.9-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + }, + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + }, + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "time": "2018-08-06T14:22:27+00:00" + }, { "name": "symfony/yaml", - "version": "v2.8.24", + "version": "v2.8.46", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", - "reference": "4c29dec8d489c4e37cf87ccd7166cd0b0e6a45c5" + "reference": "5baf0f821b14eee8ca415e6a0361a9fa140c002c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/4c29dec8d489c4e37cf87ccd7166cd0b0e6a45c5", - "reference": "4c29dec8d489c4e37cf87ccd7166cd0b0e6a45c5", + "url": "https://api.github.com/repos/symfony/yaml/zipball/5baf0f821b14eee8ca415e6a0361a9fa140c002c", + "reference": "5baf0f821b14eee8ca415e6a0361a9fa140c002c", "shasum": "" }, "require": { - "php": ">=5.3.9" + "php": ">=5.3.9", + "symfony/polyfill-ctype": "~1.8" }, "type": "library", "extra": { @@ -1085,7 +1154,7 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2017-06-01T20:52:29+00:00" + "time": "2018-08-29T13:11:53+00:00" } ], "aliases": [], From 9e8bd4db003178d8fafc386216f00743df2c04bf Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 17:54:53 -0400 Subject: [PATCH 05/17] Typo --- includes/WpMinions/SimpleQueueService/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php index 99536ca..d96d706 100644 --- a/includes/WpMinions/SimpleQueueService/Connection.php +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -8,7 +8,7 @@ class Connection { /** * Establish a connection to AWS SQS - * @throws RuntimeException is AWS PHP SDK isn't loaded + * @throws RuntimeException if AWS PHP SDK isn't loaded */ public static function connect() { if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { From 0610b3d3094d0367b4ebfcdc11e250086a2397d5 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 17:58:18 -0400 Subject: [PATCH 06/17] PHPDoc and no hard-coded value in Connection::get_queue_name --- .../WpMinions/SimpleQueueService/Connection.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php index d96d706..0679cdf 100644 --- a/includes/WpMinions/SimpleQueueService/Connection.php +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -7,7 +7,8 @@ class Connection { /** - * Establish a connection to AWS SQS + * Establish a connection to AWS SQS. + * * @throws RuntimeException if AWS PHP SDK isn't loaded */ public static function connect() { @@ -23,19 +24,19 @@ public static function connect() { } /** - * The Function Group used to split libGearman functions on a - * multi-network install. + * Builds a queue name for the async tasks. * - * @return string The prefixed group name + * @param string $baseName The unprefixed queue name + * @return string Queue name, possibly prefixed */ - public static function get_queue_name() { + public static function get_queue_name( $baseName = 'WP_Async_Task' ) { $key = ''; if ( defined( 'WP_ASYNC_TASK_SALT' ) ) { $key .= WP_ASYNC_TASK_SALT . '-'; } - $key .= 'WP_Async_Task'; + $key .= $baseName; return $key; } From 98a7154a350408e3adbafa2c9a209d345e8a92d2 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 22:05:23 -0400 Subject: [PATCH 07/17] Readme updates --- readme.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 4f88c95..2b7a348 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,7 @@ WP Minions [![Build Status](https://travis-ci.org/10up/WP-Minions.svg?branch=mas ======== Provides a framework for using job queues with [WordPress](http://wordpress.org/) for asynchronous task running. -Provides an integration with [Gearman](http://gearman.org/) and [RabbitMQ](https://www.rabbitmq.com) out of the box. +Provides an integration with [Gearman](http://gearman.org/), [RabbitMQ](https://www.rabbitmq.com), and [AWS Simple Queue Service](https://aws.amazon.com/sqs/) out of the box.

@@ -36,7 +36,7 @@ if ( ! isset( $_SERVER['HTTP_HOST'] ) && defined( 'DOING_ASYNC' ) && DOING_ASYNC } ``` -4. Next, you'll need to choose your job queue system. Gearman and RabbitMQ are supported out of the box. +4. Next, you'll need to choose your job queue system. Gearman, RabbitMQ, and AWS Simple Queue Service are supported out of the box. ### Gearman @@ -194,6 +194,32 @@ Note: For some setups, the above will not work as ```/etc/default/gearman-job-se Then restart the gearman-job-server: ```sudo service gearman-job-server restart```. +### AWS Simple Queue Service + +The Simple Queue Service requires a "key" and "secret", stored in a file. Once that is in place, we can install the WordPress plugin and set the configuration options for WordPress.\ + +#### API Credentials ("key" and "secret") + +Open [AWS IAM](https://console.aws.amazon.com/iam/home?region=us-east-1#/home) in your browser and add a user with "Programmatic access". This user needs full access to AWS SQS, which can be provided through the AmazonSQSFullAccess policy (arn:aws:iam::aws:policy/AmazonSQSFullAccess). Once the user is created, copy the "Access key ID" and "Secret access key", which you will need in the next step. + +In the home directory of the user(s) who will be running WordPress and the `wp-minions-runner.php` script, create a directory named `.aws` and a file in that directory named `credentials`. The contents of the file shoud look like this, substituting your new "Access Key ID" and "Secret access key": + +``` +[default] +aws_access_key_id=Access Key ID +aws_secret_access_key=Secret access key +``` + +#### WordPress Configuration + +Define the `WP_MINIONS_BACKEND` constant in your ```wp-config.php```. Valid values are `gearman`, `rabbitmq`, or `sqs`. If left blank, it will default to a cron client. +``` +define( 'WP_MINIONS_BACKEND', 'sqs' ); +``` + +#### Configure the wp-minions-runner script + +See the Gearman instructions for how to run wp-minions-runner.php automatically to process queued tasks. ## Verification From e1d68bfc57e1e732139a4e48b4c48af96c251312 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 22:05:45 -0400 Subject: [PATCH 08/17] Additional commenting --- includes/WpMinions/SimpleQueueService/Client.php | 1 + includes/WpMinions/SimpleQueueService/Connection.php | 3 +++ includes/WpMinions/SimpleQueueService/Worker.php | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/includes/WpMinions/SimpleQueueService/Client.php b/includes/WpMinions/SimpleQueueService/Client.php index 1ca9a2d..309d3cc 100644 --- a/includes/WpMinions/SimpleQueueService/Client.php +++ b/includes/WpMinions/SimpleQueueService/Client.php @@ -8,6 +8,7 @@ use \RuntimeException as RuntimeException; /** + * Client for adding new tasks to an AWS SQS queue. */ class Client extends BaseClient { diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php index 0679cdf..b8f3842 100644 --- a/includes/WpMinions/SimpleQueueService/Connection.php +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -4,6 +4,9 @@ use Aws\Sqs\SqsClient; +/** + * Utility methods for connecting to AWS SQS and configuring the queue. + */ class Connection { /** diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index df6fd09..90e39ce 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -6,7 +6,7 @@ use Aws\Result; /** - * + * Async task Worker implementation for AWS SQS. */ class Worker extends BaseWorker { From db6ba8b7b9bba689aef00de38d33525ef2178231 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Wed, 10 Oct 2018 22:48:34 -0400 Subject: [PATCH 09/17] Call hook appropriately in SimpleQueueService --- includes/WpMinions/SimpleQueueService/Worker.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 90e39ce..5bb71b1 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -61,7 +61,7 @@ public function work() { if( $receiveMessageResult instanceof \Aws\Result ) { $messages = $receiveMessageResult->get('Messages'); if( isset( $messages[0]['Body'] ) ) { - $payload = $messages[0]['Body']; + $payload = json_decode( $messages[0]['Body'] ); } if( isset( $messages[0]['ReceiptHandle'] ) ) { $receiptHandle = $messages[0]['ReceiptHandle']; @@ -76,11 +76,19 @@ public function work() { } } - do_action( 'wp_async_task_after_work', $payload, $this ); + if( !empty( $payload ) ) { + $hook = isset( $payload->hook ) ? $payload->hook : ''; + $args = isset( $payload->args ) ? (array) $payload->args : array(); + + if( !empty( $hook ) ) { + do_action( $hook, $args ); + do_action( 'wp_async_task_after_work', $payload, $this ); + } + } if( !empty( $payload ) && !empty( $receiptHandle ) ) { - try { + try { $callable = array( $this->sqs_client, 'deleteMessage' ); $deleteMessageResult = call_user_func( $callable, array( From b1c52452fbd30bc7b5c38cbddbbcb906e190f1a4 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Fri, 19 Oct 2018 16:24:11 -0400 Subject: [PATCH 10/17] Merged vestigial do_job() method with work() --- .../WpMinions/SimpleQueueService/Worker.php | 110 ++++++------------ 1 file changed, 38 insertions(+), 72 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 5bb71b1..2c41813 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -34,14 +34,34 @@ public function register() { } /** - * Pulls a job from the SQS Queue and tries to execute it. - * Errors are logged if the Job failed to execute. + * Executes a Job pulled from SQS. On a multisite instance + * it switches to the target site before executing the job. And the + * site is restored once executing is finished. + * + * The job data contains, + * + * 1. hook - The name of the target hook to execute + * 2. args - Optional arguments to pass to the target hook + * 3. blog_id - Optional blog on a multisite to switch to, before execution + * + * Actions are fired before and after execution of the target hook. + * + * Eg:- for the action 'foo' The order of execution of actions is, + * + * 1. wp_async_task_before_job + * 2. wp_async_task_before_job_foo + * 3. foo + * 4. wp_async_task_after_job + * 5. wp_async_task_after_job_foo * + * @param array $job The job object data. * @return bool True if the job could be executed, else false */ public function work() { + $payload = false; $receiptHandle = false; + $switched = false; try { if ( $this->sqs_client !== false ) { @@ -80,10 +100,25 @@ public function work() { $hook = isset( $payload->hook ) ? $payload->hook : ''; $args = isset( $payload->args ) ? (array) $payload->args : array(); + if ( function_exists( 'is_multisite' ) && is_multisite() && $job_data['blog_id'] ) { + $blog_id = $payload->blog_id; + + if ( get_current_blog_id() !== $blog_id ) { + switch_to_blog( $blog_id ); + $switched = true; + } + } + if( !empty( $hook ) ) { do_action( $hook, $args ); - do_action( 'wp_async_task_after_work', $payload, $this ); } + + do_action( 'wp_async_task_after_work', $payload, $this ); + + if ( $switched ) { + restore_current_blog(); + } + } if( !empty( $payload ) && !empty( $receiptHandle ) ) { @@ -109,75 +144,6 @@ public function work() { /* Helpers */ - /** - * Executes a Job pulled from SQS. On a multisite instance - * it switches to the target site before executing the job. And the - * site is restored once executing is finished. - * - * The job data contains, - * - * 1. hook - The name of the target hook to execute - * 2. args - Optional arguments to pass to the target hook - * 3. blog_id - Optional blog on a multisite to switch to, before execution - * - * Actions are fired before and after execution of the target hook. - * - * Eg:- for the action 'foo' The order of execution of actions is, - * - * 1. wp_async_task_before_job - * 2. wp_async_task_before_job_foo - * 3. foo - * 4. wp_async_task_after_job - * 5. wp_async_task_after_job_foo - * - * @param array $job The job object data. - * @return bool True or false based on the status of execution - */ - function do_job( $job ) { - $switched = false; - - try { - $job_data = json_decode( $job->workload(), true ); - $hook = $job_data['hook']; - $args = $job_data['args']; - - if ( function_exists( 'is_multisite' ) && is_multisite() && $job_data['blog_id'] ) { - $blog_id = $job_data['blog_id']; - - if ( get_current_blog_id() !== $blog_id ) { - switch_to_blog( $blog_id ); - $switched = true; - } else { - $switched = false; - } - } else { - $switched = false; - } - - do_action( 'wp_async_task_before_job', $hook, $job ); - do_action( 'wp_async_task_before_job_' . $hook, $job ); - - do_action( $hook, $args, $job ); - - do_action( 'wp_async_task_after_job', $hook, $job ); - do_action( 'wp_async_task_after_job_' . $hook, $job ); - - $result = true; - } catch ( \Exception $e ) { - - error_log( - 'SQSWorker->do_job failed: ' . $e->getMessage() - ); - $result = false; - } - - if ( $switched ) { - restore_current_blog(); - } - - return $result; - } - /** * Builds the SQS Client Instance if the extension is * installed. Once created returns the previous instance without From 1d7e60ea0081b12bbf9e8cebfc09be1741770fd2 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Mon, 22 Oct 2018 13:36:35 -0400 Subject: [PATCH 11/17] Made SQS worker run in an infinite loop per Chris M. Extracted methods. --- .../WpMinions/SimpleQueueService/Worker.php | 185 +++++++++++------- 1 file changed, 117 insertions(+), 68 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 2c41813..f5066c1 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -15,6 +15,8 @@ class Worker extends BaseWorker { */ public $sqs_client; + const DELAY_BETWEEN_ITERATIONS = 30; /* seconds */ + /** * Creates a SQS Worker and connects to AWS SQS. * The callback that will execute a job's hook is also setup here. @@ -55,91 +57,62 @@ public function register() { * 5. wp_async_task_after_job_foo * * @param array $job The job object data. - * @return bool True if the job could be executed, else false + * @return bool False if the jobs could be executed, else never returns */ public function work() { - $payload = false; - $receiptHandle = false; - $switched = false; - - try { - if ( $this->sqs_client !== false ) { - $createQueueResult = $this->sqs_client->createQueue( - array( - 'QueueName' => Connection::get_queue_name() - ) - ); - - $callable = array( $this->sqs_client, 'receiveMessage' ); - - $receiveMessageResult = call_user_func( $callable, array( - 'QueueUrl' => $createQueueResult['QueueUrl'], - 'MaxNumberOfMessages' => 1, - ) ); - - if( $receiveMessageResult instanceof \Aws\Result ) { - $messages = $receiveMessageResult->get('Messages'); - if( isset( $messages[0]['Body'] ) ) { - $payload = json_decode( $messages[0]['Body'] ); - } - if( isset( $messages[0]['ReceiptHandle'] ) ) { - $receiptHandle = $messages[0]['ReceiptHandle']; - } - } - - } + $queue = false; - } catch ( \Exception $e ) { + if( false === $this->sqs_client ) { if ( ! defined( 'PHPUNIT_RUNNER' ) ) { - error_log( 'SQSWorker failed to get message: ' . $e->getMessage() ); + error_log( 'SQSWorker could not execute: sqs_client failed to initialize' ); } + + return false; } - if( !empty( $payload ) ) { - $hook = isset( $payload->hook ) ? $payload->hook : ''; - $args = isset( $payload->args ) ? (array) $payload->args : array(); + $receiveMessageCallable = array( $this->sqs_client, 'receiveMessage' ); - if ( function_exists( 'is_multisite' ) && is_multisite() && $job_data['blog_id'] ) { - $blog_id = $payload->blog_id; + if( $queue = $this->get_queue( $this->sqs_client ) ) { + while( true ) { - if ( get_current_blog_id() !== $blog_id ) { - switch_to_blog( $blog_id ); - $switched = true; + $payload = false; + $receiptHandle = false; + + try { + $receiveMessageResult = call_user_func( $receiveMessageCallable, array( + 'QueueUrl' => $queue['QueueUrl'], + 'MaxNumberOfMessages' => 1, + ) ); + + if( $receiveMessageResult instanceof \Aws\Result ) { + $messages = $receiveMessageResult->get('Messages'); + if( isset( $messages[0]['Body'] ) ) { + $payload = json_decode( $messages[0]['Body'] ); + } + if( isset( $messages[0]['ReceiptHandle'] ) ) { + $receiptHandle = $messages[0]['ReceiptHandle']; + } + } + } catch ( \Exception $e ) { + if ( ! defined( 'PHPUNIT_RUNNER' ) ) { + error_log( 'SQSWorker failed to get message: ' . $e->getMessage() ); + } } - } - - if( !empty( $hook ) ) { - do_action( $hook, $args ); - } - - do_action( 'wp_async_task_after_work', $payload, $this ); - if ( $switched ) { - restore_current_blog(); - } - - } - - if( !empty( $payload ) && !empty( $receiptHandle ) ) { - - try { - $callable = array( $this->sqs_client, 'deleteMessage' ); - - $deleteMessageResult = call_user_func( $callable, array( - 'QueueUrl' => $createQueueResult['QueueUrl'], - 'ReceiptHandle' => $receiptHandle, - ) ); + if( !empty( $payload ) ) { + $this->process_payload( $payload ); + } - } - catch ( \Exception $e ) { - if ( ! defined( 'PHPUNIT_RUNNER' ) ) { - error_log( 'SQSWorker failed to delete message: ' . $e->getMessage() ); + if( !empty( $receiptHandle ) ) { + $this->delete_message( $queue, $receiptHandle ); } + + sleep( self::DELAY_BETWEEN_ITERATIONS ); } } - return $payload !== false; + return false; } /* Helpers */ @@ -159,4 +132,80 @@ function get_sqs_client() { return $this->sqs_client; } + /** + * + */ + function get_queue( $sqs_client ) { + + $queue = false; + + try { + $queue = $sqs_client->createQueue( + array( + 'QueueName' => Connection::get_queue_name() + ) + ); + } catch ( \Exception $e ) { + if ( ! defined( 'PHPUNIT_RUNNER' ) ) { + error_log( 'SQSWorker failed to create queue: ' . $e->getMessage() ); + } + } + + return $queue; + } + + /** + * @param stdClass $payload + * @return void + */ + function process_payload( $payload ) { + $hook = isset( $payload->hook ) ? $payload->hook : ''; + $args = isset( $payload->args ) ? (array) $payload->args : array(); + + $switched = false; + + if ( function_exists( 'is_multisite' ) && is_multisite() && !empty( $payload->blog_id ) ) { + $blog_id = $payload->blog_id; + + if ( get_current_blog_id() !== $blog_id ) { + switch_to_blog( $blog_id ); + $switched = true; + } + } + + if( !empty( $hook ) ) { + do_action( $hook, $args ); + } + + do_action( 'wp_async_task_after_work', $payload, $this ); + + if ( $switched ) { + restore_current_blog(); + } + } + + /** + * @return \Aws\Result|false + */ + function delete_message( $queue, $receiptHandle ) { + + $deleteMessageCallable = array( $this->sqs_client, 'deleteMessage' ); + $deleteMessageResult = false; + + try { + $deleteMessageResult = call_user_func( $deleteMessageCallable, array( + 'QueueUrl' => $queue['QueueUrl'], + 'ReceiptHandle' => $receiptHandle, + ) ); + + } + catch ( \Exception $e ) { + if ( ! defined( 'PHPUNIT_RUNNER' ) ) { + error_log( 'SQSWorker failed to delete message: ' . $e->getMessage() ); + } + } + + return $deleteMessageResult; + } + } From aa22978450e1a1bb9af04a80fc18f034bd132066 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Mon, 22 Oct 2018 13:41:59 -0400 Subject: [PATCH 12/17] Loop every 5s instead of 30s --- includes/WpMinions/SimpleQueueService/Worker.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index f5066c1..9b41f23 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -15,7 +15,7 @@ class Worker extends BaseWorker { */ public $sqs_client; - const DELAY_BETWEEN_ITERATIONS = 30; /* seconds */ + const DELAY_BETWEEN_ITERATIONS = 5; /* seconds */ /** * Creates a SQS Worker and connects to AWS SQS. From 45a87aa5973f49abd11333c039be4c56ddeddf9f Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Mon, 22 Oct 2018 13:52:54 -0400 Subject: [PATCH 13/17] Improvements to SQS integration's delete_message method --- includes/WpMinions/SimpleQueueService/Worker.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 9b41f23..0a2b614 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -105,7 +105,7 @@ public function work() { } if( !empty( $receiptHandle ) ) { - $this->delete_message( $queue, $receiptHandle ); + $this->delete_message( $queue['QueueUrl'], $receiptHandle ); } sleep( self::DELAY_BETWEEN_ITERATIONS ); @@ -185,16 +185,19 @@ function process_payload( $payload ) { } /** - * @return \Aws\Result|false + * Delete a message from an SQS queue + * + * @param string $queueURL Queue URL + * @param string $receiptHandle Unique message receipt handle + * @return Aws\Result|false */ - function delete_message( $queue, $receiptHandle ) { - + function delete_message( $queueURL, $receiptHandle ) { $deleteMessageCallable = array( $this->sqs_client, 'deleteMessage' ); $deleteMessageResult = false; try { $deleteMessageResult = call_user_func( $deleteMessageCallable, array( - 'QueueUrl' => $queue['QueueUrl'], + 'QueueUrl' => $queueURL, 'ReceiptHandle' => $receiptHandle, ) ); From 7446825751a7079e2563256753883b10a7064de0 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Mon, 22 Oct 2018 14:16:35 -0400 Subject: [PATCH 14/17] PHPDoc --- includes/WpMinions/SimpleQueueService/Worker.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 0a2b614..9a1c984 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -108,7 +108,6 @@ public function work() { $this->delete_message( $queue['QueueUrl'], $receiptHandle ); } - sleep( self::DELAY_BETWEEN_ITERATIONS ); } } @@ -133,10 +132,11 @@ function get_sqs_client() { } /** - * + * Get the information to connect to an SQS queue + * @param Aws\Sqs\SqsClient $sqs_client + * @return Aws\Result queue information, URL in [QueueUrl] value */ function get_queue( $sqs_client ) { - $queue = false; try { @@ -155,6 +155,8 @@ function get_queue( $sqs_client ) { } /** + * Process the payload from an SQS queue + * * @param stdClass $payload * @return void */ From 85113950a31fe45263ca1fe6cfc62dde60042e98 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Mon, 22 Oct 2018 16:03:31 -0400 Subject: [PATCH 15/17] Put the sleep() command back in SQS Worker --- includes/WpMinions/SimpleQueueService/Worker.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/includes/WpMinions/SimpleQueueService/Worker.php b/includes/WpMinions/SimpleQueueService/Worker.php index 9a1c984..93ed0a6 100644 --- a/includes/WpMinions/SimpleQueueService/Worker.php +++ b/includes/WpMinions/SimpleQueueService/Worker.php @@ -103,6 +103,10 @@ public function work() { if( !empty( $payload ) ) { $this->process_payload( $payload ); } + else { + // Sleep to let the server rest before checking the queue again. + sleep( self::DELAY_BETWEEN_ITERATIONS ); + } if( !empty( $receiptHandle ) ) { $this->delete_message( $queue['QueueUrl'], $receiptHandle ); From 39173598cdc0828b313baa097e588223626d59f7 Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Tue, 23 Oct 2018 10:56:57 -0400 Subject: [PATCH 16/17] Allow configuration through $awssqs_server global --- .../SimpleQueueService/Connection.php | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/includes/WpMinions/SimpleQueueService/Connection.php b/includes/WpMinions/SimpleQueueService/Connection.php index b8f3842..6a186cc 100644 --- a/includes/WpMinions/SimpleQueueService/Connection.php +++ b/includes/WpMinions/SimpleQueueService/Connection.php @@ -15,12 +15,27 @@ class Connection { * @throws RuntimeException if AWS PHP SDK isn't loaded */ public static function connect() { + global $awssqs_server; + if ( class_exists( 'Aws\Sqs\SqsClient' ) ) { - return SqsClient::factory(array( + $clientConfig = array( 'version' => '2012-11-05', - 'profile' => self::get_profile_name(), 'region' => self::get_region_name(), - )); + ); + + if( !empty( $awssqs_server ) ) { + if( isset( $awssqs_server['access_key'] ) && isset( $awssqs_server['secret'] ) ) { + $clientConfig['credentials'] = array( + 'key' => $awssqs_server['access_key'], + 'secret' => $awssqs_server['secret'], + ); + } + } + else { + $clientConfig['profile'] = self::get_profile_name(); + } + + return SqsClient::factory( $clientConfig ); } else { throw new RuntimeException('AWS SDK not loaded'); } @@ -52,7 +67,12 @@ public static function get_queue_name( $baseName = 'WP_Async_Task' ) { * @return string region name */ public static function get_region_name( $default = 'us-east-1' ) { - if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { + global $awssqs_server; + + if( isset( $awssqs_server['region'] ) ) { + return $awssqs_server['region']; + } + else if( isset( $_ENV['AWS_DEFAULT_REGION '] ) ) { return $_ENV['AWS_DEFAULT_REGION ']; } else { @@ -68,7 +88,12 @@ public static function get_region_name( $default = 'us-east-1' ) { * @return string profile name */ public static function get_profile_name( $default = 'default' ) { - if( isset( $_ENV['AWS_PROFILE'] ) ) { + global $awssqs_server; + + if( isset( $awssqs_server['profile'] ) ) { + return $awssqs_server['profile']; + } + else if( isset( $_ENV['AWS_PROFILE'] ) ) { return $_ENV['AWS_PROFILE']; } else { From 2c9628a51773eddd5e80abe86ec988e26326076e Mon Sep 17 00:00:00 2001 From: Dave Ross Date: Tue, 23 Oct 2018 11:23:33 -0400 Subject: [PATCH 17/17] Notes on configuring SQS integration through wp-config.php --- readme.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/readme.md b/readme.md index 2b7a348..feb95c3 100644 --- a/readme.md +++ b/readme.md @@ -202,6 +202,23 @@ The Simple Queue Service requires a "key" and "secret", stored in a file. Once t Open [AWS IAM](https://console.aws.amazon.com/iam/home?region=us-east-1#/home) in your browser and add a user with "Programmatic access". This user needs full access to AWS SQS, which can be provided through the AmazonSQSFullAccess policy (arn:aws:iam::aws:policy/AmazonSQSFullAccess). Once the user is created, copy the "Access key ID" and "Secret access key", which you will need in the next step. +There are multiple ways to provide AWS credentials to WP Minions: + +##### wp-config.php + +The most straightforward way to connect WP Minions with AWS is to define your AWS credentials in ```wp-config.php```: + +```php +global $awssqs_server; +$awssqs_server = array( + 'access_key' => Access Key ID, + 'secret' => Secret access key, + 'region' => Region, // optional +); +``` + +##### .aws Directory + In the home directory of the user(s) who will be running WordPress and the `wp-minions-runner.php` script, create a directory named `.aws` and a file in that directory named `credentials`. The contents of the file shoud look like this, substituting your new "Access Key ID" and "Secret access key": ``` @@ -210,6 +227,27 @@ aws_access_key_id=Access Key ID aws_secret_access_key=Secret access key ``` +Multiple profiles can be defined in `.aws/credentials`, for example: + +``` +[default] +aws_access_key_id=Access Key ID +aws_secret_access_key=Secret access key + +[10up] +aws_access_key_id=Access Key ID +aws_secret_access_key=Secret access key +``` + +Then, specify which profile to use in `wp-config.php`: + +```php +global $awssqs_server; +$awssqs_server = array( + 'profile' => '10up', // example +); +``` + #### WordPress Configuration Define the `WP_MINIONS_BACKEND` constant in your ```wp-config.php```. Valid values are `gearman`, `rabbitmq`, or `sqs`. If left blank, it will default to a cron client.