From 8592e0d7aae93bf080aee61aa9148f2741b4c192 Mon Sep 17 00:00:00 2001 From: Gerrit Giliomee Date: Fri, 6 May 2022 11:34:49 +0200 Subject: [PATCH 1/5] func(IPFS): Try retrieving image from own node if we don't get a response from IPFS. --- app/Http/Controllers/ImageController.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/ImageController.php b/app/Http/Controllers/ImageController.php index b90075a..b8d0f24 100644 --- a/app/Http/Controllers/ImageController.php +++ b/app/Http/Controllers/ImageController.php @@ -126,8 +126,17 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 $response = $client->request('GET', $url, ['connect_timeout' => 10, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); if ($response->getStatusCode() !== 200) { - // Something went wrong, so remove what was downloaded by $client->request, sink - unlink(storage_path() . $storeLocation); + /** + * If we can't get a response from IPFS, try getting a response from our own node. + */ + $backupUrl = str_replace('https://ipfs.io/ipfs/', 'http://139.59.103.146:8080/ipfs/', $url); + $client = new \GuzzleHttp\Client(); + $response = $client->request('GET', $backupUrl, ['connect_timeout' => 10, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); + + if ($response->getStatusCode() !== 200) { + // Something went wrong, so remove what was downloaded by $client->request, sink + unlink(storage_path() . $storeLocation); + } } Cache::forget($cacheName); From 44678c789649a4ef3c251c8067a2df4640d698d0 Mon Sep 17 00:00:00 2001 From: Gerrit Giliomee Date: Fri, 13 May 2022 11:52:30 +0200 Subject: [PATCH 2/5] func(Gateway timeout): Add additional check to remove the downloaded response in the event of a gateway timeout. --- app/Http/Controllers/ImageController.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/ImageController.php b/app/Http/Controllers/ImageController.php index b8d0f24..3c57ff6 100644 --- a/app/Http/Controllers/ImageController.php +++ b/app/Http/Controllers/ImageController.php @@ -123,17 +123,20 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 // Get a local copy of the image to work with asap and ensure multiple requests to download a local copy don't happen Cache::put($cacheName, true, 30); $client = new \GuzzleHttp\Client(); - $response = $client->request('GET', $url, ['connect_timeout' => 10, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); + $response = $client->request('GET', $url, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); + + if (intval($response->getStatusCode()) !== 200) { + // Remove the downloaded response, as it's invalid + unlink(storage_path() . $storeLocation); - if ($response->getStatusCode() !== 200) { /** * If we can't get a response from IPFS, try getting a response from our own node. */ $backupUrl = str_replace('https://ipfs.io/ipfs/', 'http://139.59.103.146:8080/ipfs/', $url); $client = new \GuzzleHttp\Client(); - $response = $client->request('GET', $backupUrl, ['connect_timeout' => 10, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); + $response = $client->request('GET', $backupUrl, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); - if ($response->getStatusCode() !== 200) { + if (intval($response->getStatusCode()) !== 200) { // Something went wrong, so remove what was downloaded by $client->request, sink unlink(storage_path() . $storeLocation); } From 636f89458c8422e490a70cb22282a248df4923a9 Mon Sep 17 00:00:00 2001 From: Gerrit Giliomee Date: Fri, 13 May 2022 11:58:56 +0200 Subject: [PATCH 3/5] func(Original image): Remove if the original image looks suspiciously small in terms of file size. --- app/Http/Controllers/ImageController.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/ImageController.php b/app/Http/Controllers/ImageController.php index 3c57ff6..9f901d1 100644 --- a/app/Http/Controllers/ImageController.php +++ b/app/Http/Controllers/ImageController.php @@ -15,7 +15,8 @@ class ImageController extends Controller { - public $maxNrTimesToTryAndProcessAnImage = 3; + private $maxNrTimesToTryAndProcessAnImage = 3; // How many times to retry the processing of an image + private $minimumSizeOfOriginalImage = 165; // The minimum size of an original image. Below this size, remove the original so that a retry occurs. /** * Takes a .gif url, width, height and quality as URL parameters. @@ -114,7 +115,12 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 $cacheName = 'ImageController::downloadOriginalImageOrWait-' . md5($storeLocation); if (file_exists(storage_path() . $storeLocation)) { - return; + if (filesize(storage_path() . $storeLocation) < $this->minimumSizeOfOriginalImage) { + // If our original image size is very small, it indicates a potential error, so remove + unlink(storage_path() . $storeLocation); + } else { + return; + } } for ($i = 1; $i <= $retries; $i++) { From 2ad5ce65d4edf573f2acbba9e0d379407d0d89da Mon Sep 17 00:00:00 2001 From: Gerrit Giliomee Date: Fri, 13 May 2022 12:41:50 +0200 Subject: [PATCH 4/5] func(Gateway timeout): Improve error handling and cleanup of invalid responses. --- app/Http/Controllers/ImageController.php | 61 +++++++++++++----------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/app/Http/Controllers/ImageController.php b/app/Http/Controllers/ImageController.php index 9f901d1..f2a3296 100644 --- a/app/Http/Controllers/ImageController.php +++ b/app/Http/Controllers/ImageController.php @@ -15,8 +15,8 @@ class ImageController extends Controller { - private $maxNrTimesToTryAndProcessAnImage = 3; // How many times to retry the processing of an image - private $minimumSizeOfOriginalImage = 165; // The minimum size of an original image. Below this size, remove the original so that a retry occurs. + private $maxNrTimesToTryAndProcessAnImage = 3; // + private $minimumSizeOfOriginalImage = 180; // The minimum size of an original image. Below this size, remove the original so that a retry occurs. /** * Takes a .gif url, width, height and quality as URL parameters. @@ -112,15 +112,13 @@ private function getImageIdentifier($url) */ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60, $retries = 3) { + // Use cache to prevent multiple requests to the same image $cacheName = 'ImageController::downloadOriginalImageOrWait-' . md5($storeLocation); + $this->removeOriginalImageIfItsTooSmall($storeLocation); + if (file_exists(storage_path() . $storeLocation)) { - if (filesize(storage_path() . $storeLocation) < $this->minimumSizeOfOriginalImage) { - // If our original image size is very small, it indicates a potential error, so remove - unlink(storage_path() . $storeLocation); - } else { - return; - } + return; } for ($i = 1; $i <= $retries; $i++) { @@ -129,25 +127,7 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 // Get a local copy of the image to work with asap and ensure multiple requests to download a local copy don't happen Cache::put($cacheName, true, 30); $client = new \GuzzleHttp\Client(); - $response = $client->request('GET', $url, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); - - if (intval($response->getStatusCode()) !== 200) { - // Remove the downloaded response, as it's invalid - unlink(storage_path() . $storeLocation); - - /** - * If we can't get a response from IPFS, try getting a response from our own node. - */ - $backupUrl = str_replace('https://ipfs.io/ipfs/', 'http://139.59.103.146:8080/ipfs/', $url); - $client = new \GuzzleHttp\Client(); - $response = $client->request('GET', $backupUrl, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); - - if (intval($response->getStatusCode()) !== 200) { - // Something went wrong, so remove what was downloaded by $client->request, sink - unlink(storage_path() . $storeLocation); - } - } - + $client->request('GET', $url, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); Cache::forget($cacheName); } else { // Wait until we have an original message @@ -160,10 +140,37 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 } break; } catch (Exception $e) { + $this->removeOriginalImageIfItsTooSmall($storeLocation); + + try { + /** + * If we can't get a response from IPFS, try getting a response from our own node. + */ + $backupUrl = str_replace('https://ipfs.io/ipfs/', 'http://139.59.103.146:8080/ipfs/', $url); + $client = new \GuzzleHttp\Client(); + $client->request('GET', $backupUrl, ['connect_timeout' => 30, 'sink' => storage_path() . $storeLocation, 'synchronous' => true]); + } catch (Exception $e) { + $this->removeOriginalImageIfItsTooSmall($storeLocation); + } + + Cache::forget($cacheName); } } } + /** + * The way that requests are made, we might end up with an invalid response, e.g. gateway timeout that is stored as a file. In such cases, the file stored is suspiciously small. Remove if needed. + */ + private function removeOriginalImageIfItsTooSmall($storeLocation) + { + if (filesize(storage_path() . $storeLocation) <= $this->minimumSizeOfOriginalImage) { + // If our original image size is very small, it indicates a potential error, so remove + unlink(storage_path() . $storeLocation); + return true; + } + return false; + } + private function resizeImageFromOriginalIfNeeded(Image $image) { // Don't get stuck processing unprocessable images From c810a70fddf896b4cf950f06a06b4b0875d459d1 Mon Sep 17 00:00:00 2001 From: Gerrit Giliomee Date: Fri, 13 May 2022 13:05:59 +0200 Subject: [PATCH 5/5] fix(Filesize): Check that file exists before trying to determine size. --- app/Http/Controllers/ImageController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/ImageController.php b/app/Http/Controllers/ImageController.php index f2a3296..8b324e2 100644 --- a/app/Http/Controllers/ImageController.php +++ b/app/Http/Controllers/ImageController.php @@ -163,7 +163,7 @@ private function downloadOriginalImageOrWait($url, $storeLocation, $seconds = 60 */ private function removeOriginalImageIfItsTooSmall($storeLocation) { - if (filesize(storage_path() . $storeLocation) <= $this->minimumSizeOfOriginalImage) { + if (file_exists(storage_path() . $storeLocation) && filesize(storage_path() . $storeLocation) <= $this->minimumSizeOfOriginalImage) { // If our original image size is very small, it indicates a potential error, so remove unlink(storage_path() . $storeLocation); return true;