From ac837c1b4f0d94b4709b2010ec08c24731eb7b4a Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Mon, 20 Sep 2021 13:34:05 +0800 Subject: [PATCH 01/14] Add media item DB table migration, initial model --- .../2021_10_02_000026_Db_Media_Items.php | 40 +++++++++++++++++++ modules/system/models/MediaItem.php | 16 ++++++++ 2 files changed, 56 insertions(+) create mode 100644 modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php create mode 100644 modules/system/models/MediaItem.php diff --git a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php new file mode 100644 index 0000000000..a303d3b638 --- /dev/null +++ b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php @@ -0,0 +1,40 @@ +engine = 'InnoDB'; + $table->increments('id'); + $table->integer('parent_id')->unsigned(); + $table->integer('nest_left')->unsigned(); + $table->integer('nest_right')->unsigned(); + $table->integer('nest_depth')->unsigned(); + $table->string('type', 20)->nullable(); + $table->string('file_type', 20)->nullable(); + $table->string('name'); + $table->string('extension', 20)->nullable(); + $table->string('path')->nullable(); + $table->bigInteger('size')->unsigned(); + $table->integer('width')->unsigned()->nullable(); + $table->integer('height')->unsigned()->nullable(); + $table->integer('duration')->unsigned()->nullable(); + $table->dateTime('modified_at')->nullable(); + + // Indexes + $table->index(['parent_id'], 'media_idx_parent_id'); + $table->index(['type'], 'media_idx_type'); + $table->index(['file_type'], 'media_idx_file_type'); + }); + } + + public function down() + { + Schema::drop('media_items'); + } +} diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php new file mode 100644 index 0000000000..ff1055cf2f --- /dev/null +++ b/modules/system/models/MediaItem.php @@ -0,0 +1,16 @@ + Date: Mon, 20 Sep 2021 16:47:46 +0800 Subject: [PATCH 02/14] Fix migration --- .../migrations/2021_10_02_000026_Db_Media_Items.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php index a303d3b638..e658d51f4e 100644 --- a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php +++ b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php @@ -11,10 +11,10 @@ public function up() // Schema $table->engine = 'InnoDB'; $table->increments('id'); - $table->integer('parent_id')->unsigned(); - $table->integer('nest_left')->unsigned(); - $table->integer('nest_right')->unsigned(); - $table->integer('nest_depth')->unsigned(); + $table->integer('parent_id')->unsigned()->nullable(); + $table->integer('nest_left')->unsigned()->nullable(); + $table->integer('nest_right')->unsigned()->nullable(); + $table->integer('nest_depth')->unsigned()->nullable(); $table->string('type', 20)->nullable(); $table->string('file_type', 20)->nullable(); $table->string('name'); From 81e9b1da30feb248b30e10a5f9cde64ddddcb768 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Mon, 20 Sep 2021 16:49:17 +0800 Subject: [PATCH 03/14] Initial work on media library scanning --- modules/backend/widgets/MediaManager.php | 17 ++ .../mediamanager/assets/js/mediamanager.js | 14 ++ .../widgets/mediamanager/partials/_body.htm | 1 + .../mediamanager/partials/_scan-popup.htm | 31 +++ modules/system/classes/MediaLibrary.php | 191 +++++++++++------- modules/system/models/MediaItem.php | 184 +++++++++++++++++ 6 files changed, 367 insertions(+), 71 deletions(-) create mode 100644 modules/backend/widgets/mediamanager/partials/_scan-popup.htm diff --git a/modules/backend/widgets/MediaManager.php b/modules/backend/widgets/MediaManager.php index 6e49b8b779..86d4094ab0 100644 --- a/modules/backend/widgets/MediaManager.php +++ b/modules/backend/widgets/MediaManager.php @@ -18,6 +18,7 @@ use Winter\Storm\Database\Attach\Resizer; use Winter\Storm\Filesystem\Definitions as FileDefinitions; use Form as FormHelper; +use System\Models\Parameter; /** * Media Manager widget. @@ -814,6 +815,21 @@ public function onResizeImage() return $this->getCropEditImageUrlAndSize($path, $cropSessionKey, $params); } + /** + * Executed when the media library has not yet been scanned, or the user opts to manually re-scan the media library. + * + * @return void + */ + public function onScan() + { + return $this->makePartial('scan-popup'); + } + + public function onScanExecute() + { + $contents = MediaLibrary::instance()->scan(); + } + // // Methods for internal use // @@ -841,6 +857,7 @@ protected function prepareVars() $this->vars['items'] = $this->findFiles($searchTerm, $filter, ['by' => $sortBy, 'direction' => $sortDirection]); } + $this->vars['isScanned'] = !is_null(Parameter::get('media::scan.last_scanned')); $this->vars['currentFolder'] = $folder; $this->vars['isRootFolder'] = $folder == self::FOLDER_ROOT; $this->vars['pathSegments'] = $this->splitPathToSegments($folder); diff --git a/modules/backend/widgets/mediamanager/assets/js/mediamanager.js b/modules/backend/widgets/mediamanager/assets/js/mediamanager.js index 7c64e47dfc..0ddb6ede3d 100644 --- a/modules/backend/widgets/mediamanager/assets/js/mediamanager.js +++ b/modules/backend/widgets/mediamanager/assets/js/mediamanager.js @@ -129,6 +129,10 @@ this.generateThumbnails() this.initUploader() this.initScroll() + + if (!this.options.isScanned) { + this.doScan() + } } MediaManager.prototype.registerHandlers = function() { @@ -1279,6 +1283,16 @@ } } + // + // Media scanning + // + + MediaManager.prototype.doScan = function() { + $.popup({ + handler: 'onScan' + }); + } + // MEDIA MANAGER PLUGIN DEFINITION // ============================ diff --git a/modules/backend/widgets/mediamanager/partials/_body.htm b/modules/backend/widgets/mediamanager/partials/_body.htm index 3dd1b17e0a..650b91aa97 100644 --- a/modules/backend/widgets/mediamanager/partials/_body.htm +++ b/modules/backend/widgets/mediamanager/partials/_body.htm @@ -11,6 +11,7 @@ data-bottom-toolbar="bottomToolbar ? 'true' : 'false' ?>" data-crop-and-insert-button="cropAndInsertButton ? 'true' : 'false' ?>" data-read-only="readOnly ? 'true' : 'false'; ?>" + data-is-scanned="isScanned ? 'true' : 'false'; ?>" tabindex="0" > diff --git a/modules/backend/widgets/mediamanager/partials/_scan-popup.htm b/modules/backend/widgets/mediamanager/partials/_scan-popup.htm new file mode 100644 index 0000000000..09cb04e16a --- /dev/null +++ b/modules/backend/widgets/mediamanager/partials/_scan-popup.htm @@ -0,0 +1,31 @@ +
+
+
+
+ +
+

 

+
+
+ Scanning media library... +
+ +
+
+

 

+
+ + diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 589b6e30c6..50e506afa1 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -9,7 +9,9 @@ use Url; use Winter\Storm\Filesystem\Definitions as FileDefinitions; use ApplicationException; +use System\Models\MediaItem; use SystemException; +use Winter\Storm\Argon\Argon; /** * Provides abstraction level for the Media Library operations. @@ -111,56 +113,13 @@ public function getCacheKey() * @param boolean $ignoreFolders Determines whether folders should be suppressed in the result list. * @return array Returns an array of MediaLibraryItem objects. */ - public function listFolderContents($folder = '/', $sortBy = 'title', $filter = null, $ignoreFolders = false) + public function listFolderContents($path = '/', $sortBy = 'title', $filter = null, $ignoreFolders = false) { - $folder = self::validatePath($folder); - $fullFolderPath = $this->getMediaPath($folder); - - /* - * Try to load the contents from cache - */ - - $cached = Cache::get($this->cacheKey, false); - $cached = $cached ? @unserialize(@base64_decode($cached)) : []; - - if (!is_array($cached)) { - $cached = []; - } - - if (array_key_exists($fullFolderPath, $cached)) { - $folderContents = $cached[$fullFolderPath]; - } - else { - $folderContents = $this->scanFolderContents($fullFolderPath); - - $cached[$fullFolderPath] = $folderContents; - $expiresAt = now()->addMinutes(Config::get('cms.storage.media.ttl', 10)); - Cache::put( - $this->cacheKey, - base64_encode(serialize($cached)), - $expiresAt - ); - } - - /* - * Sort the result and combine the file and folder lists - */ - - if ($sortBy !== false) { - $this->sortItemList($folderContents['files'], $sortBy); - $this->sortItemList($folderContents['folders'], $sortBy); - } - - $this->filterItemList($folderContents['files'], $filter); - - if (!$ignoreFolders) { - $folderContents = array_merge($folderContents['folders'], $folderContents['files']); - } - else { - $folderContents = $folderContents['files']; + if (!$folder = MediaItem::folder($path)) { + return []; } - - return $folderContents; + return []; + // return $folder->contents($sortBy, $filter, $ignoreFolders); } /** @@ -551,6 +510,118 @@ public function getPathUrl($path) } } + /** + * Scans the disk and stores all metadata in the "media_items" table for performant traversing and filtering. + * + * This will clear the "media_items" table. + * + * @return void + */ + public function scan(MediaItem $folder = null, $path = null) + { + if ($folder === null) { + MediaItem::truncate(); + $contents = $this->getStorageDisk()->listContents($this->getMediaPath('/')); + $folder = MediaItem::getRoot(); + } else { + $contents = $this->getStorageDisk()->listContents($path); + } + + // Filter contents so that ignored filenames and patterns are applied + $contents = array_filter($contents, function ($item) { + return $this->isVisible($item['path']); + }); + + foreach ($contents as $item) { + if ($item['type'] === 'dir') { + $subFolder = $this->createFolderMeta($folder, $item); + $this->scan($subFolder, $item['path']); + continue; + } + + $this->createFileMeta($folder, $item); + } + } + + /** + * Creates a meta record for a folder in the "media_items" table, as a subfolder of the parent folder. + * + * @param MediaItem $parent + * @param array $meta + * @return MediaItem + */ + protected function createFolderMeta(MediaItem $parent, array $meta) + { + return $parent->children()->create([ + 'name' => $meta['filename'], + 'path' => ltrim($this->getMediaRelativePath($meta['path']), '/'), + 'type' => MediaLibraryItem::TYPE_FOLDER, + 'size' => 0, + 'modified_at' => Argon::createFromTimestamp($meta['timestamp']), + ]); + } + + /** + * Creates a meta record for a file in the "media_items" table, as a child file of the parent folder. + * + * @param MediaItem $parent + * @param array $meta + * @return MediaItem + */ + protected function createFileMeta(MediaItem $parent, array $meta) + { + // Create a temporary media library item instance + $path = ltrim($this->getMediaRelativePath($meta['path']), '/'); + $mediaItem = new MediaLibraryItem( + $path, + $meta['size'], + $meta['timestamp'], + MediaLibraryItem::TYPE_FILE, + $this->getPathUrl($path) + ); + + // Standard metadata + $file = $parent->children()->make([ + 'name' => $meta['filename'], + 'path' => $path, + 'type' => MediaLibraryItem::TYPE_FILE, + 'extension' => $meta['extension'], + 'size' => $meta['size'], + 'modified_at' => Argon::createFromTimestamp($meta['timestamp']), + ]); + + // Extra metadata + $file->file_type = $mediaItem->getFileType(); + + if ($mediaItem->getFileType() === MediaLibraryItem::FILE_TYPE_IMAGE) { + $this->setImageMeta($file, $meta['path']); + } + + $file->save(); + + return $file; + } + + /** + * Gets meta for images. + * + * This scans the image for the dimensions and stores them in the meta table. + * + * @param MediaItem $file + * @param string $path + * @return void + */ + protected function setImageMeta(MediaItem $file, $path) + { + $size = @getimagesizefromstring($this->getStorageDisk()->read($path)); + if (!is_array($size)) { + return; + } + + $file->width = $size[0]; + $file->height = $size[1]; + } + /** * Returns a file or folder path with the prefixed storage folder. * @param string $path Specifies a path to process. @@ -745,34 +816,12 @@ protected function sortItemList(&$itemList, $sortSettings) }); } - /** - * Filters item list by file type. - * @param array $itemList Specifies the item list to sort. - * @param string $filter Determines the document type filtering preference. - * Supported values are 'image', 'video', 'audio', 'document' (see FILE_TYPE_XXX constants of MediaLibraryItem class). - */ - protected function filterItemList(&$itemList, $filter) - { - if (!$filter) { - return; - } - - $result = []; - foreach ($itemList as $item) { - if ($item->getFileType() == $filter) { - $result[] = $item; - } - } - - $itemList = $result; - } - /** * Initializes and returns the Media Library disk. * This method should always be used instead of trying to access the * $storageDisk property directly as initializing the disc requires * communicating with the remote storage. - * @return mixed Returns the storage disk object. + * @return \Illuminate\Contracts\Filesystem\Filesystem Returns the storage disk object. */ protected function getStorageDisk() { diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index ff1055cf2f..d54e7afaca 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -2,6 +2,7 @@ use Model; use System\Classes\MediaLibraryItem; +use Winter\Storm\Database\Relations\HasMany; class MediaItem extends Model { @@ -13,4 +14,187 @@ class MediaItem extends Model * @var string */ public $table = 'media_items'; + + /** + * Disable timestamps + * + * @var boolean + */ + public $timestamps = false; + + /** + * Fillable attributes. + * + * @var array + */ + public $fillable = [ + 'type', + 'name', + 'path', + 'extension', + 'size', + 'modified_at', + ]; + + /** + * Cache of the root node. + * + * @var static + */ + protected static $rootCache; + + /** + * Finds a folder at the specific path. + * + * @param string $path + * @return static + */ + public static function folder($path = '/') + { + if ($path === '/' || empty($path)) { + return self::getRoot(); + } + + $query = self::query(); + $path = trim($path, '/'); + + $query + ->where('type', MediaLibraryItem::TYPE_FOLDER) + ->where('path', $path); + + return $query->firstOrFail(); + } + + /** + * Retrieves the contents of a folder. + * + * @param string|array $sortBy + * @param string|null $filter + * @param boolean $ignoreFolders + * @return array + */ + public function contents($sortBy = 'title', $filter = null, $ignoreFolders = false) + { + $query = $this->children(); + + $query = $this->applySort($query, $sortBy); + if (!is_null($filter)) { + $query = $this->applyFilter($query, $filter); + } + + if ($ignoreFolders) { + $query->where('type', MediaLibraryItem::TYPE_FILE); + } + + // Group by type + return $query + ->get() + ->toArray(); + } + + /** + * Applies sorting to the contents of a folder. + * + * @param HasMany $query + * @param string|array $sortBy Accepts either a string value of "title", "size", or "lastModified", or an array + * with two keys: "by" which represents the sort type (same 3 options as above), and "direction" - either "asc" + * or "desc". + * @return HasMany + */ + protected function applySort(HasMany $query, $sortBy) + { + // Force ordering of folders first + $query->orderBy('type', 'DESC'); + + if (is_array($sortBy)) { + $sortDir = $sortBy['direction'] ?? 'asc'; + $sortBy = $sortBy['by']; + } else { + $sortDir = 'asc'; + } + + if (!in_array($sortBy, ['title', 'size', 'lastModified'])) { + return $query; + } + + switch ($sortBy) { + case 'title': + $query->orderBy('name', $sortDir); + break; + case 'size': + $query->orderBy('size', $sortDir); + break; + case 'lastModified': + $query->orderBy('modified_at', $sortDir); + break; + } + + return $query; + } + + /** + * Applies a filter to the contents of the directory. This only applies to files, not folders. + * + * @param HasMany $query + * @param string $filter Accepts one of the following: "audio", "document", "image", "video" + * @return HasMany + */ + protected function applyFilter(HasMany $query, $filter) + { + if (!in_array($filter, [ + MediaLibraryItem::FILE_TYPE_AUDIO, + MediaLibraryItem::FILE_TYPE_DOCUMENT, + MediaLibraryItem::FILE_TYPE_IMAGE, + MediaLibraryItem::FILE_TYPE_VIDEO, + ])) { + return $query; + } + + return $query->where(function ($query) use ($filter) { + $query->where('file_type', $filter) + ->orWhere('type', MediaLibraryItem::TYPE_FOLDER); + }); + } + + /** + * Creates an empty root node to represent the root folder of the media library. + * + * @return static + */ + protected static function makeRoot() + { + $root = new static; + $root->parent_id = null; + $root->type = MediaLibraryItem::TYPE_FOLDER; + $root->name = 'Root'; + $root->size = 0; + $root->save(); + + return $root; + } + + /** + * Gets the root folder, or creates one if it does not exist. + * + * @return static + */ + public static function getRoot() + { + if (self::$rootCache) { + return self::$rootCache; + } + + $query = self::query(); + + // Find the root folder node + $query + ->whereNull('parent_id'); + + if (!$query->count()) { + // Create a root node if one does not already exist + return self::$rootCache = self::makeRoot(); + } + + return self::$rootCache = $query->first(); + } } From 9789e9cf1dcc023b74ca3db7af33057c000f0bf8 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Mon, 20 Sep 2021 22:24:18 +0800 Subject: [PATCH 04/14] Use synchronisation method for scans --- modules/system/classes/MediaLibrary.php | 62 ++++++++++++++++++++++--- modules/system/models/MediaItem.php | 23 +++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 50e506afa1..6267af465c 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -67,6 +67,11 @@ class MediaLibrary */ protected $storageFolderNameLength; + /** + * @var array Scanned meta, used to compare a subsequent scan to act only on changes. + */ + protected $scannedMeta; + /** * Initialize this singleton. */ @@ -513,14 +518,33 @@ public function getPathUrl($path) /** * Scans the disk and stores all metadata in the "media_items" table for performant traversing and filtering. * - * This will clear the "media_items" table. + * Scanning, by default, will be done in a synchronisation fashion - only metadata that needs to be updated + * will be updated, in order to keep subsequent scans quicker. It does this by tracking the path and the + * modification time, however, you may opt to force a full resync using the `$forceResync` parameter. + * + * @param MediaItem $folder The root media folder for this iteration. If `null`, the system assumes the root. + * @param string|null $path The root path of this folder. + * @param bool $forceResync If `true`, a full resync is done by truncating the "media_items" table. * * @return void */ - public function scan(MediaItem $folder = null, $path = null) + public function scan(MediaItem $folder = null, $path = null, $forceResync = false) { - if ($folder === null) { - MediaItem::truncate(); + $isRoot = is_null($folder); + + if ($isRoot) { + if ($forceResync) { + MediaItem::truncate(); + } + + $this->scannedMeta = MediaItem::notRoot() + ->get() + ->pluck('modified_at', 'path') + ->map(function ($item) { + return Argon::parse($item)->getTimestamp(); + }) + ->toArray(); + $contents = $this->getStorageDisk()->listContents($this->getMediaPath('/')); $folder = MediaItem::getRoot(); } else { @@ -533,13 +557,39 @@ public function scan(MediaItem $folder = null, $path = null) }); foreach ($contents as $item) { + $mediaPath = ltrim($this->getMediaRelativePath($item['path']), '/'); + if ($item['type'] === 'dir') { - $subFolder = $this->createFolderMeta($folder, $item); + // Determine if we are adding a new directory + if (!isset($this->scannedMeta[$mediaPath])) { + $subFolder = $this->createFolderMeta($folder, $item); + } else { + $subFolder = MediaItem::where('path', $mediaPath)->first(); + unset($this->scannedMeta[$mediaPath]); + } + $this->scan($subFolder, $item['path']); continue; } - $this->createFileMeta($folder, $item); + if (!isset($this->scannedMeta[$mediaPath])) { + // New file detected + $this->createFileMeta($folder, $item); + continue; + } elseif ($this->scannedMeta[$mediaPath] < $item['timestamp']) { + // File was modified + MediaItem::where('path', $mediaPath)->delete(); + $this->createFileMeta($folder, $item); + } + + unset($this->scannedMeta[$mediaPath]); + } + + // Any scanned meta still in the list when we have looped through the root files are now deleted + if ($isRoot && count($this->scannedMeta)) { + foreach (array_keys($this->scannedMeta) as $path) { + MediaItem::where('path', $path)->delete(); + } } } diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index d54e7afaca..29aa84cad0 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -2,6 +2,7 @@ use Model; use System\Classes\MediaLibraryItem; +use Winter\Storm\Database\Builder; use Winter\Storm\Database\Relations\HasMany; class MediaItem extends Model @@ -43,6 +44,28 @@ class MediaItem extends Model */ protected static $rootCache; + /** + * Scope to get the root folder of the media library. + * + * @param Builder $query + * @return void + */ + public function scopeRoot(Builder $query) + { + $query->whereNull('parent_id'); + } + + /** + * Scope to get all but the root folder of the media library. + * + * @param Builder $query + * @return void + */ + public function scopeNotRoot(Builder $query) + { + $query->whereNotNull('parent_id'); + } + /** * Finds a folder at the specific path. * From 428e2408845119925440476ad41dc3d306081e11 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Mon, 20 Sep 2021 22:47:36 +0800 Subject: [PATCH 05/14] Initial preview of media browsing running from DB --- modules/backend/widgets/MediaManager.php | 14 ++++++++----- .../widgets/mediamanager/partials/_body.htm | 2 +- modules/system/classes/MediaLibrary.php | 15 +++++++++----- modules/system/models/MediaItem.php | 20 +++++++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/modules/backend/widgets/MediaManager.php b/modules/backend/widgets/MediaManager.php index 86d4094ab0..6463ce1246 100644 --- a/modules/backend/widgets/MediaManager.php +++ b/modules/backend/widgets/MediaManager.php @@ -1,14 +1,11 @@ scan(); + MediaLibrary::instance()->scan(); + + $this->prepareVars(); + + return [ + '#'.$this->getId('item-list') => $this->makePartial('item-list'), + '#'.$this->getId('folder-path') => $this->makePartial('folder-path'), + '#'.$this->getId('filters') => $this->makePartial('filters') + ]; } // diff --git a/modules/backend/widgets/mediamanager/partials/_body.htm b/modules/backend/widgets/mediamanager/partials/_body.htm index 650b91aa97..2c2631159d 100644 --- a/modules/backend/widgets/mediamanager/partials/_body.htm +++ b/modules/backend/widgets/mediamanager/partials/_body.htm @@ -11,7 +11,7 @@ data-bottom-toolbar="bottomToolbar ? 'true' : 'false' ?>" data-crop-and-insert-button="cropAndInsertButton ? 'true' : 'false' ?>" data-read-only="readOnly ? 'true' : 'false'; ?>" - data-is-scanned="isScanned ? 'true' : 'false'; ?>" + data-is-scanned="" tabindex="0" > diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 6267af465c..d5e3cc7dbe 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -5,13 +5,13 @@ use Cache; use Config; use Storage; -use Request; use Url; -use Winter\Storm\Filesystem\Definitions as FileDefinitions; use ApplicationException; -use System\Models\MediaItem; use SystemException; +use System\Models\MediaItem; +use System\Models\Parameter; use Winter\Storm\Argon\Argon; +use Winter\Storm\Filesystem\Definitions as FileDefinitions; /** * Provides abstraction level for the Media Library operations. @@ -123,8 +123,8 @@ public function listFolderContents($path = '/', $sortBy = 'title', $filter = nul if (!$folder = MediaItem::folder($path)) { return []; } - return []; - // return $folder->contents($sortBy, $filter, $ignoreFolders); + + return $folder->contents($sortBy, $filter, $ignoreFolders); } /** @@ -591,6 +591,11 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals MediaItem::where('path', $path)->delete(); } } + + // Update last scan parameter + if ($isRoot) { + Parameter::set('media::scan.last_scanned', Argon::now()); + } } /** diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 29aa84cad0..2e04a9379f 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -1,6 +1,7 @@ get() + ->map(function ($item) { + return $item->toMediaLibraryItem(); + }) ->toArray(); } + /** + * Converts a media item model instance into a MediaLibraryItem instance. + * + * @return MediaLibraryItem + */ + protected function toMediaLibraryItem() + { + return new MediaLibraryItem( + $this->path, + $this->size, + Argon::parse($this->modified_at)->getTimestamp(), + $this->type, + '', + ); + } + /** * Applies sorting to the contents of a folder. * From b1e0b992c6384fb7e97e49be45056788f99974ec Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Tue, 21 Sep 2021 15:08:19 +0800 Subject: [PATCH 06/14] Add metadata array in model, add method to set metadata, fix tests --- modules/system/classes/MediaLibrary.php | 34 ++++++-- .../2021_10_02_000026_Db_Media_Items.php | 4 +- modules/system/lang/en/lang.php | 3 + modules/system/models/MediaItem.php | 77 ++++++++++++++++++- .../unit/system/classes/MediaLibraryTest.php | 20 +++-- 5 files changed, 121 insertions(+), 17 deletions(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index d5e3cc7dbe..346c233dd5 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -568,7 +568,9 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals unset($this->scannedMeta[$mediaPath]); } - $this->scan($subFolder, $item['path']); + if (!is_null($subFolder)) { + $this->scan($subFolder, $item['path']); + } continue; } @@ -607,9 +609,15 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals */ protected function createFolderMeta(MediaItem $parent, array $meta) { + try { + $path = self::validatePath($meta['path']); + } catch (ApplicationException $e) { + return; + } + return $parent->children()->create([ 'name' => $meta['filename'], - 'path' => ltrim($this->getMediaRelativePath($meta['path']), '/'), + 'path' => $this->getMediaRelativePath($path), 'type' => MediaLibraryItem::TYPE_FOLDER, 'size' => 0, 'modified_at' => Argon::createFromTimestamp($meta['timestamp']), @@ -625,8 +633,14 @@ protected function createFolderMeta(MediaItem $parent, array $meta) */ protected function createFileMeta(MediaItem $parent, array $meta) { + try { + $path = self::validatePath($meta['path']); + } catch (ApplicationException $e) { + return; + } + $path = $this->getMediaRelativePath($path); + // Create a temporary media library item instance - $path = ltrim($this->getMediaRelativePath($meta['path']), '/'); $mediaItem = new MediaLibraryItem( $path, $meta['size'], @@ -673,8 +687,18 @@ protected function setImageMeta(MediaItem $file, $path) return; } - $file->width = $size[0]; - $file->height = $size[1]; + $file->setMetadata([ + 'width' => [ + 'label' => 'system::lang.media.metadata_image_width', + 'order' => 200, + 'value' => $size[0], + ], + 'height' => [ + 'label' => 'system::lang.media.metadata_image_height', + 'order' => 220, + 'value' => $size[1], + ] + ]); } /** diff --git a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php index e658d51f4e..cc3bbb9204 100644 --- a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php +++ b/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php @@ -21,9 +21,7 @@ public function up() $table->string('extension', 20)->nullable(); $table->string('path')->nullable(); $table->bigInteger('size')->unsigned(); - $table->integer('width')->unsigned()->nullable(); - $table->integer('height')->unsigned()->nullable(); - $table->integer('duration')->unsigned()->nullable(); + $table->text('metadata')->nullable(); $table->dateTime('modified_at')->nullable(); // Indexes diff --git a/modules/system/lang/en/lang.php b/modules/system/lang/en/lang.php index 40c70ed940..1bd5062248 100644 --- a/modules/system/lang/en/lang.php +++ b/modules/system/lang/en/lang.php @@ -466,6 +466,9 @@ 'media' => [ 'invalid_path' => "Invalid file path specified: ':path'.", 'folder_size_items' => 'item(s)', + 'metadata_image_width' => 'Width', + 'metadata_image_height' => 'Height', + 'metadata_video_duration' => 'Duration', ], 'page' => [ 'custom_error' => [ diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 2e04a9379f..90dccbc57d 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -38,6 +38,24 @@ class MediaItem extends Model 'modified_at', ]; + /** + * Arrayable attributes. + * + * @var array + */ + public $jsonable = [ + 'metadata', + ]; + + /** + * Date attributes. + * + * @var array + */ + public $dates = [ + 'modified_at', + ]; + /** * Cache of the root node. * @@ -67,6 +85,24 @@ public function scopeNotRoot(Builder $query) $query->whereNotNull('parent_id'); } + /** + * Override of base `setAttribute` method. + * + * Prevents writing to metadata key directly. + * + * @param string $key + * @param mixed $value + * @return mixed + */ + public function setAttribute($key, $value) + { + if ($key === 'metadata') { + return false; + } + + return parent::setAttribute($key, $value); + } + /** * Finds a folder at the specific path. * @@ -129,12 +165,49 @@ protected function toMediaLibraryItem() return new MediaLibraryItem( $this->path, $this->size, - Argon::parse($this->modified_at)->getTimestamp(), + $this->modified_at->getTimestamp(), $this->type, - '', + '' ); } + /** + * Sets metadata for this media item. + * + * @param string|array $key + * @param mixed $value + * @param string|null $label + * @param integer $order + * @param string $group + * @return void + */ + public function setMetadata($key, $value = null, $label = null, $order = 100, $group = 'Metadata') + { + if (is_array($key)) { + foreach ($key as $name => $options) { + $this->setMetadata( + $name, + $options['value'] ?? null, + $options['label'] ?? $name, + $options['order'] ?? 100, + $options['group'] ?? 'Metadata', + ); + } + return; + } + + $meta = array_replace($this->metadata ?? [], [ + $key => [ + 'label' => $label ?? $key, + 'order' => $order, + 'group' => $group, + 'value' => $value, + ] + ]); + + parent::setAttribute('metadata', $meta); + } + /** * Applies sorting to the contents of a folder. * diff --git a/tests/unit/system/classes/MediaLibraryTest.php b/tests/unit/system/classes/MediaLibraryTest.php index 0dbafe3f95..0607a51f2d 100644 --- a/tests/unit/system/classes/MediaLibraryTest.php +++ b/tests/unit/system/classes/MediaLibraryTest.php @@ -3,7 +3,7 @@ use Illuminate\Filesystem\FilesystemAdapter; use System\Classes\MediaLibrary; -class MediaLibraryTest extends TestCase // @codingStandardsIgnoreLine +class MediaLibraryTest extends PluginTestCase // @codingStandardsIgnoreLine { public function setUp(): void { @@ -80,20 +80,26 @@ public function testListFolderContents() { $this->setUpStorage(); $this->copyMedia(); - + MediaLibrary::instance()->scan(); $contents = MediaLibrary::instance()->listFolderContents(); + $this->assertNotEmpty($contents, 'Media library item is not discovered'); $this->assertCount(3, $contents); - $this->assertEquals('file', $contents[2]->type, 'Media library item does not have the right type'); - $this->assertEquals('/winter.png', $contents[2]->path, 'Media library item does not have the right path'); - $this->assertNotEmpty($contents[2]->lastModified, 'Media library item last modified is empty'); - $this->assertNotEmpty($contents[2]->size, 'Media library item size is empty'); - $this->assertEquals('file', $contents[0]->type, 'Media library item does not have the right type'); $this->assertEquals('/text.txt', $contents[0]->path, 'Media library item does not have the right path'); $this->assertNotEmpty($contents[0]->lastModified, 'Media library item last modified is empty'); $this->assertNotEmpty($contents[0]->size, 'Media library item size is empty'); + + $this->assertEquals('file', $contents[1]->type, 'Media library item does not have the right type'); + $this->assertEquals('/winter.png', $contents[1]->path, 'Media library item does not have the right path'); + $this->assertNotEmpty($contents[1]->lastModified, 'Media library item last modified is empty'); + $this->assertNotEmpty($contents[1]->size, 'Media library item size is empty'); + + $this->assertEquals('file', $contents[2]->type, 'Media library item does not have the right type'); + $this->assertEquals('/winter space.png', $contents[2]->path, 'Media library item does not have the right path'); + $this->assertNotEmpty($contents[2]->lastModified, 'Media library item last modified is empty'); + $this->assertNotEmpty($contents[2]->size, 'Media library item size is empty'); } public function testListAllDirectories() From b6218896ddd0bbcd83d948b55b5691e1a6a60d7e Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Tue, 21 Sep 2021 15:17:05 +0800 Subject: [PATCH 07/14] Remove trailing slash - PHP 7.2 doesn't support them for params --- modules/system/models/MediaItem.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 90dccbc57d..116d2aeee4 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -190,7 +190,7 @@ public function setMetadata($key, $value = null, $label = null, $order = 100, $g $options['value'] ?? null, $options['label'] ?? $name, $options['order'] ?? 100, - $options['group'] ?? 'Metadata', + $options['group'] ?? 'Metadata' ); } return; From 5eb93cf1449a356aaf5799928fcfa2173e241821 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Tue, 21 Sep 2021 15:38:42 +0800 Subject: [PATCH 08/14] Implement search --- modules/system/classes/MediaLibrary.php | 103 +----------------------- modules/system/models/MediaItem.php | 53 ++++++++++-- 2 files changed, 47 insertions(+), 109 deletions(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 346c233dd5..679c0660bb 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -138,33 +138,7 @@ public function listFolderContents($path = '/', $sortBy = 'title', $filter = nul */ public function findFiles($searchTerm, $sortBy = 'title', $filter = null) { - $words = explode(' ', Str::lower($searchTerm)); - $result = []; - - $findInFolder = function ($folder) use (&$findInFolder, $words, &$result, $sortBy, $filter) { - $folderContents = $this->listFolderContents($folder, $sortBy, $filter); - - foreach ($folderContents as $item) { - if ($item->type == MediaLibraryItem::TYPE_FOLDER) { - $findInFolder($item->path); - } - elseif ($this->pathMatchesSearch($item->path, $words)) { - $result[] = $item; - } - } - }; - - $findInFolder('/'); - - /* - * Sort the result - */ - - if ($sortBy !== false) { - $this->sortItemList($result, $sortBy); - } - - return $result; + return MediaItem::getRoot()->search($searchTerm, $sortBy, $filter); } /** @@ -844,57 +818,6 @@ protected function scanFolderContents($fullFolderPath) return $result; } - /** - * Sorts the item list by title, size or last modified date. - * @param array $itemList Specifies the item list to sort. - * @param mixed $sortSettings Determines the sorting preference. - * Supported values are 'title', 'size', 'lastModified' (see SORT_BY_XXX class constants) or an associative array with a 'by' key and a 'direction' key: ['by' => SORT_BY_XXX, 'direction' => SORT_DIRECTION_XXX]. - */ - protected function sortItemList(&$itemList, $sortSettings) - { - $files = []; - $folders = []; - - // Convert string $sortBy to array - if (is_string($sortSettings)) { - $sortSettings = [ - 'by' => $sortSettings, - 'direction' => self::SORT_DIRECTION_ASC, - ]; - } - - usort($itemList, function ($a, $b) use ($sortSettings) { - $result = 0; - - switch ($sortSettings['by']) { - case self::SORT_BY_TITLE: - $result = strcasecmp($a->path, $b->path); - break; - case self::SORT_BY_SIZE: - if ($a->size < $b->size) { - $result = -1; - } else { - $result = $a->size > $b->size ? 1 : 0; - } - break; - case self::SORT_BY_MODIFIED: - if ($a->lastModified < $b->lastModified) { - $result = -1; - } else { - $result = $a->lastModified > $b->lastModified ? 1 : 0; - } - break; - } - - // Reverse the polarity of the result to direct sorting in a descending order instead - if ($sortSettings['direction'] === self::SORT_DIRECTION_DESC) { - $result = 0 - $result; - } - - return $result; - }); - } - /** * Initializes and returns the Media Library disk. * This method should always be used instead of trying to access the @@ -913,30 +836,6 @@ protected function getStorageDisk() ); } - /** - * Determines if file path contains all words form the search term. - * @param string $path Specifies a path to examine. - * @param array $words A list of words to check against. - * @return boolean - */ - protected function pathMatchesSearch($path, $words) - { - $path = Str::lower($path); - - foreach ($words as $word) { - $word = trim($word); - if (!strlen($word)) { - continue; - } - - if (!Str::contains($path, $word)) { - return false; - } - } - - return true; - } - protected function generateRandomTmpFolderName($location) { $temporaryDirBaseName = time(); diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 116d2aeee4..cb57a50362 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -1,7 +1,7 @@ toArray(); } + /** + * Searches within a folder recursively for files or folders that match a search query. + * + * @param string|array $sortBy + * @param string|null $filter + * @param boolean $ignoreFolders + * @return array + */ + public function search($searchTerm, $sortBy = 'title', $filter = null) + { + // Normalise search term(s) + $words = explode(' ', Str::lower($searchTerm)); + + $query = $this->allChildren(); + + // Only return files as results + $query->where('type', MediaLibraryItem::TYPE_FILE); + + $query = $this->applySort($query, $sortBy); + if (!is_null($filter)) { + $query = $this->applyFilter($query, $filter); + } + + // Add search query + $query->where(function ($query) use ($words) { + foreach ($words as $word) { + $query->orWhere('path', 'like', '%' . $word . '%'); + } + }); + + // Group by type + return $query + ->get() + ->map(function ($item) { + return $item->toMediaLibraryItem(); + }) + ->toArray(); + } + /** * Converts a media item model instance into a MediaLibraryItem instance. * @@ -211,13 +250,13 @@ public function setMetadata($key, $value = null, $label = null, $order = 100, $g /** * Applies sorting to the contents of a folder. * - * @param HasMany $query + * @param HasMany|Builder $query * @param string|array $sortBy Accepts either a string value of "title", "size", or "lastModified", or an array * with two keys: "by" which represents the sort type (same 3 options as above), and "direction" - either "asc" * or "desc". - * @return HasMany + * @return HasMany|Builder */ - protected function applySort(HasMany $query, $sortBy) + protected function applySort($query, $sortBy) { // Force ordering of folders first $query->orderBy('type', 'DESC'); @@ -251,11 +290,11 @@ protected function applySort(HasMany $query, $sortBy) /** * Applies a filter to the contents of the directory. This only applies to files, not folders. * - * @param HasMany $query + * @param HasMany|Builder $query * @param string $filter Accepts one of the following: "audio", "document", "image", "video" - * @return HasMany + * @return HasMany|Builder */ - protected function applyFilter(HasMany $query, $filter) + protected function applyFilter($query, $filter) { if (!in_array($filter, [ MediaLibraryItem::FILE_TYPE_AUDIO, From c085e4f48f284f9494e4152c6515ee33fde8a811 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Tue, 21 Sep 2021 16:43:57 +0800 Subject: [PATCH 09/14] Add directory list, bug fixes --- modules/system/classes/MediaLibrary.php | 38 ++++------------------ modules/system/models/MediaItem.php | 43 +++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 679c0660bb..5eae2da060 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -10,6 +10,7 @@ use SystemException; use System\Models\MediaItem; use System\Models\Parameter; +use Illuminate\Database\Eloquent\ModelNotFoundException; use Winter\Storm\Argon\Argon; use Winter\Storm\Filesystem\Definitions as FileDefinitions; @@ -120,7 +121,9 @@ public function getCacheKey() */ public function listFolderContents($path = '/', $sortBy = 'title', $filter = null, $ignoreFolders = false) { - if (!$folder = MediaItem::folder($path)) { + try { + $folder = MediaItem::folder($path); + } catch (ModelNotFoundException $e) { return []; } @@ -212,36 +215,9 @@ public function folderExists($path) */ public function listAllDirectories($exclude = []) { - $fullPath = $this->getMediaPath('/'); - - $folders = $this->getStorageDisk()->allDirectories($fullPath); - - $folders = array_unique($folders, SORT_LOCALE_STRING); - - $result = []; - - foreach ($folders as $folder) { - $folder = $this->getMediaRelativePath($folder); - if (!strlen($folder)) { - $folder = '/'; - } - - if (Str::startsWith($folder, $exclude)) { - continue; - } - if (!$this->isVisible($folder)) { - $exclude[] = $folder . '/'; - continue; - } - - $result[] = $folder; - } - - if (!in_array('/', $result)) { - array_unshift($result, '/'); - } - - return $result; + return array_map(function ($item) { + return $item->path; + }, MediaItem::getRoot()->folders($exclude)); } /** diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index cb57a50362..740e9e0b98 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -116,7 +116,6 @@ public static function folder($path = '/') } $query = self::query(); - $path = trim($path, '/'); $query ->where('type', MediaLibraryItem::TYPE_FOLDER) @@ -128,6 +127,8 @@ public static function folder($path = '/') /** * Retrieves the contents of a folder. * + * This includes both files and folders. + * * @param string|array $sortBy * @param string|null $filter * @param boolean $ignoreFolders @@ -146,7 +147,6 @@ public function contents($sortBy = 'title', $filter = null, $ignoreFolders = fal $query->where('type', MediaLibraryItem::TYPE_FILE); } - // Group by type return $query ->get() ->map(function ($item) { @@ -155,6 +155,42 @@ public function contents($sortBy = 'title', $filter = null, $ignoreFolders = fal ->toArray(); } + + /** + * Retrieves the folders within a folder. + * + * You may optionally provide an array of paths to exclude. + * + * @param array $exclude + * @param bool $includeSelf + * @return array + */ + public function folders($exclude = [], $includeSelf = true) + { + $query = $this->allChildren()->where('type', MediaLibraryItem::TYPE_FOLDER); + + $directories = $query->get(); + + if ($includeSelf) { + $directories->prepend($this); + } + + return $directories + ->filter(function ($item) use ($exclude) { + foreach ($exclude as $path) { + if (Str::startsWith($item->path, $exclude)) { + return false; + } + } + + return true; + }) + ->map(function ($item) { + return $item->toMediaLibraryItem(); + }) + ->toArray(); + } + /** * Searches within a folder recursively for files or folders that match a search query. * @@ -204,7 +240,7 @@ protected function toMediaLibraryItem() return new MediaLibraryItem( $this->path, $this->size, - $this->modified_at->getTimestamp(), + (isset($this->modified_at)) ? $this->modified_at->getTimestamp() : null, $this->type, '' ); @@ -322,6 +358,7 @@ protected static function makeRoot() $root->parent_id = null; $root->type = MediaLibraryItem::TYPE_FOLDER; $root->name = 'Root'; + $root->path = '/'; $root->size = 0; $root->save(); From c11ac265c3cdd36d6a35a0c108c4e452911c5bd4 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Tue, 21 Sep 2021 17:11:11 +0800 Subject: [PATCH 10/14] Fix potential error --- modules/system/classes/MediaLibrary.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 5eae2da060..1e1b80e75c 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -502,7 +502,7 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals } // Filter contents so that ignored filenames and patterns are applied - $contents = array_filter($contents, function ($item) { + $contents = array_filter($contents ?? [], function ($item) { return $this->isVisible($item['path']); }); From 8da227141638b888acb6b1217e8f82b1b2a0b17f Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Sat, 25 Sep 2021 08:40:09 +0800 Subject: [PATCH 11/14] Further fixes to tests --- modules/system/classes/MediaLibrary.php | 5 +- .../unit/system/classes/MediaLibraryTest.php | 164 ++++++++++++++++-- 2 files changed, 149 insertions(+), 20 deletions(-) diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 1e1b80e75c..aad0a1d88f 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -495,7 +495,8 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals }) ->toArray(); - $contents = $this->getStorageDisk()->listContents($this->getMediaPath('/')); + $rootMedia = $this->getMediaPath('/'); + $contents = $this->getStorageDisk()->listContents($rootMedia); $folder = MediaItem::getRoot(); } else { $contents = $this->getStorageDisk()->listContents($path); @@ -507,7 +508,7 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals }); foreach ($contents as $item) { - $mediaPath = ltrim($this->getMediaRelativePath($item['path']), '/'); + $mediaPath = $this->getMediaRelativePath($item['path']); if ($item['type'] === 'dir') { // Determine if we are adding a new directory diff --git a/tests/unit/system/classes/MediaLibraryTest.php b/tests/unit/system/classes/MediaLibraryTest.php index 0607a51f2d..615897b619 100644 --- a/tests/unit/system/classes/MediaLibraryTest.php +++ b/tests/unit/system/classes/MediaLibraryTest.php @@ -1,7 +1,8 @@ setUpStorage(); $this->copyMedia(); - MediaLibrary::instance()->scan(); + + // Rescan library + MediaLibrary::instance()->scan(null, null, true); + $contents = MediaLibrary::instance()->listFolderContents(); $this->assertNotEmpty($contents, 'Media library item is not discovered'); @@ -104,21 +108,142 @@ public function testListFolderContents() public function testListAllDirectories() { - $disk = $this->createConfiguredMock(FilesystemAdapter::class, [ - 'allDirectories' => [ - '/media/.ignore1', - '/media/.ignore2', - '/media/dir', - '/media/dir/sub', - '/media/exclude', - '/media/hidden', - '/media/hidden/sub1', - '/media/hidden/sub1/deep1', - '/media/hidden/sub2', - '/media/hidden but not really', - '/media/name' - ] - ]); + $disk = $this->createMock(FilesystemInterface::class); + + $disk->expects($this->any()) + ->method('listContents') + ->willReturnCallback(function ($path) { + switch ($path) { + case '/media/': + return [ + [ + 'type' => 'dir', + 'path' => 'media/.ignore1', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => '.ignore1', + 'extension' => '', + 'filename' => '.ignore1' + ], + [ + 'type' => 'dir', + 'path' => 'media/.ignore2', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => '.ignore2', + 'extension' => '', + 'filename' => '.ignore2' + ], + [ + 'type' => 'dir', + 'path' => 'media/dir', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => 'dir', + 'extension' => '', + 'filename' => 'dir' + ], + [ + 'type' => 'dir', + 'path' => 'media/exclude', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => 'exclude', + 'extension' => '', + 'filename' => 'exclude' + ], + [ + 'type' => 'dir', + 'path' => 'media/hidden', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => 'hidden', + 'extension' => '', + 'filename' => 'hidden' + ], + [ + 'type' => 'dir', + 'path' => 'media/hidden but not really', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media', + 'basename' => 'hidden but not really', + 'extension' => '', + 'filename' => 'hidden but not really' + ], + [ + 'type' => 'file', + 'path' => 'media/name', + 'timestamp' => now(), + 'size' => 24, + 'dirname' => 'media', + 'basename' => 'name', + 'extension' => '', + 'filename' => 'name' + ], + ]; + break; + case '/media/dir': + return [ + [ + 'type' => 'dir', + 'path' => 'media/dir/sub', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media/dir', + 'basename' => 'sub', + 'extension' => '', + 'filename' => 'sub' + ], + ]; + break; + case '/media/hidden': + return [ + [ + 'type' => 'dir', + 'path' => 'media/hidden/sub1', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media/hidden', + 'basename' => 'sub1', + 'extension' => '', + 'filename' => 'sub1' + ], + [ + 'type' => 'dir', + 'path' => 'media/hidden/sub2', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media/hidden', + 'basename' => 'sub2', + 'extension' => '', + 'filename' => 'sub2' + ], + ]; + break; + case '/media/hidden/sub1': + return [ + [ + 'type' => 'dir', + 'path' => 'media/hidden/sub1/deep1', + 'timestamp' => now(), + 'size' => 0, + 'dirname' => 'media/hidden/sub1', + 'basename' => 'deep1', + 'extension' => '', + 'filename' => 'deep1' + ], + ]; + break; + default: + return []; + } + }); $this->app['config']->set('cms.storage.media.folder', 'media'); $this->app['config']->set('cms.storage.media.ignore', ['hidden']); @@ -126,7 +251,10 @@ public function testListAllDirectories() $instance = MediaLibrary::instance(); $this->setProtectedProperty($instance, 'storageDisk', $disk); - $this->assertEquals(['/', '/dir', '/dir/sub', '/hidden but not really', '/name'], $instance->listAllDirectories(['/exclude'])); + // Rescan library + MediaLibrary::instance()->scan(null, null, true); + + $this->assertEquals(['/', '/dir', '/dir/sub', '/hidden but not really'], $instance->listAllDirectories(['/exclude'])); } protected function setUpStorage() From 2bc991f1030566a638591357fb65e84ca85911cf Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Thu, 31 Aug 2023 14:53:37 +0800 Subject: [PATCH 12/14] Update migration, use path enumeration for media items --- composer.json | 2 +- ...b_Media_Items.php => 2023_08_31_000028_Db_Media_Items.php} | 4 +--- modules/system/models/MediaItem.php | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) rename modules/system/database/migrations/{2021_10_02_000026_Db_Media_Items.php => 2023_08_31_000028_Db_Media_Items.php} (84%) diff --git a/composer.json b/composer.json index 553c7ddb97..04f02a42bc 100644 --- a/composer.json +++ b/composer.json @@ -30,7 +30,7 @@ }, "require": { "php": "^8.0.2", - "winter/storm": "dev-develop as 1.2", + "winter/storm": "dev-wip/path-enumerable-trait as 1.2", "winter/wn-system-module": "dev-develop", "winter/wn-backend-module": "dev-develop", "winter/wn-cms-module": "dev-develop", diff --git a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php b/modules/system/database/migrations/2023_08_31_000028_Db_Media_Items.php similarity index 84% rename from modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php rename to modules/system/database/migrations/2023_08_31_000028_Db_Media_Items.php index cc3bbb9204..51152b3b5d 100644 --- a/modules/system/database/migrations/2021_10_02_000026_Db_Media_Items.php +++ b/modules/system/database/migrations/2023_08_31_000028_Db_Media_Items.php @@ -12,9 +12,7 @@ public function up() $table->engine = 'InnoDB'; $table->increments('id'); $table->integer('parent_id')->unsigned()->nullable(); - $table->integer('nest_left')->unsigned()->nullable(); - $table->integer('nest_right')->unsigned()->nullable(); - $table->integer('nest_depth')->unsigned()->nullable(); + $table->string('location', 192)->nullable(); $table->string('type', 20)->nullable(); $table->string('file_type', 20)->nullable(); $table->string('name'); diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 740e9e0b98..43794bbdf1 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -8,7 +8,8 @@ class MediaItem extends Model { - use \Winter\Storm\Database\Traits\NestedTree; + use \Winter\Storm\Database\Traits\PathEnumerable; + const PATH_COLUMN = 'location'; /** * Table to use. From bcbcd6868b82a9e760c7fb7444c8349eb9c6f696 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Thu, 31 Aug 2023 15:33:21 +0800 Subject: [PATCH 13/14] Fix logic to work more gracefully with Flysystem --- .../codeeditor/assets/js/build-min.js | 6 +- .../assets/js/mediamanager-browser-min.js | 4 +- modules/system/classes/MediaLibrary.php | 58 ++++++++----------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/modules/backend/formwidgets/codeeditor/assets/js/build-min.js b/modules/backend/formwidgets/codeeditor/assets/js/build-min.js index e1dd79a93e..4a6c376976 100644 --- a/modules/backend/formwidgets/codeeditor/assets/js/build-min.js +++ b/modules/backend/formwidgets/codeeditor/assets/js/build-min.js @@ -1290,9 +1290,9 @@ getData=function(row){return popup.data[row];};popup.getRow=function(){return se z-index: 2;\ }\ .ace_editor.ace_autocomplete .ace_scroller {\ - background: none;\ - border: none;\ - box-shadow: none;\ + background: none;\ + border: none;\ + box-shadow: none;\ }\ .ace_rightAlignedText {\ color: gray;\ diff --git a/modules/backend/widgets/mediamanager/assets/js/mediamanager-browser-min.js b/modules/backend/widgets/mediamanager/assets/js/mediamanager-browser-min.js index 4ec9f0e50d..66d599b7ac 100644 --- a/modules/backend/widgets/mediamanager/assets/js/mediamanager-browser-min.js +++ b/modules/backend/widgets/mediamanager/assets/js/mediamanager-browser-min.js @@ -53,7 +53,8 @@ if(this.options.cropAndInsertButton)this.$el.find('[data-popup-command="crop-and this.updateSidebarPreview() this.generateThumbnails() this.initUploader() -this.initScroll()} +this.initScroll() +if(!this.options.isScanned){this.doScan()}} MediaManager.prototype.registerHandlers=function(){this.$el.on('dblclick',this.proxy(this.onNavigate)) this.$el.on('click.tree-path','ul.tree-path, [data-control="sidebar-labels"]',this.proxy(this.onNavigate)) this.$el.on('click.command','[data-command]',this.proxy(this.onCommandClick)) @@ -358,6 +359,7 @@ break;case'ArrowLeft':case'ArrowUp':this.selectRelative(false,ev.shiftKey) eventHandled=true break;}if(eventHandled){ev.preventDefault() ev.stopPropagation()}} +MediaManager.prototype.doScan=function(){$.popup({handler:'onScan'});} MediaManager.DEFAULTS={url:window.location,uploadHandler:null,alias:'',deleteEmpty:'Please select files to delete.',deleteConfirm:'Delete the selected file(s)?',moveEmpty:'Please select files to move.',selectSingleImage:'Please select a single image.',selectionNotImage:'The selected item is not an image.',bottomToolbar:false,cropAndInsertButton:false} var old=$.fn.mediaManager $.fn.mediaManager=function(option){var args=Array.prototype.slice.call(arguments,1),result=undefined diff --git a/modules/system/classes/MediaLibrary.php b/modules/system/classes/MediaLibrary.php index 7ebfca67eb..37afcdb4f0 100644 --- a/modules/system/classes/MediaLibrary.php +++ b/modules/system/classes/MediaLibrary.php @@ -6,13 +6,16 @@ use Config; use Storage; use Url; -use ApplicationException; -use SystemException; use System\Models\MediaItem; use System\Models\Parameter; use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Database\Eloquent\ModelNotFoundException; +use League\Flysystem\DirectoryAttributes; +use League\Flysystem\FileAttributes; +use League\Flysystem\StorageAttributes; use Winter\Storm\Argon\Argon; +use Winter\Storm\Exception\ApplicationException ; +use Winter\Storm\Exception\SystemException; use Winter\Storm\Filesystem\Definitions as FileDefinitions; /** @@ -504,14 +507,15 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals } // Filter contents so that ignored filenames and patterns are applied - $contents = array_filter($contents ?? [], function ($item) { - return $this->isVisible($item['path']); + $contents = $contents->filter(function (StorageAttributes $item) { + return $this->isVisible($item->path()); }); + /** @var StorageAttributes $item */ foreach ($contents as $item) { $mediaPath = $this->getMediaRelativePath($item['path']); - if ($item['type'] === 'dir') { + if ($item->type() === 'dir') { // Determine if we are adding a new directory if (!isset($this->scannedMeta[$mediaPath])) { $subFolder = $this->createFolderMeta($folder, $item); @@ -530,7 +534,7 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals // New file detected $this->createFileMeta($folder, $item); continue; - } elseif ($this->scannedMeta[$mediaPath] < $item['timestamp']) { + } elseif ($this->scannedMeta[$mediaPath] < $item->timestamp) { // File was modified MediaItem::where('path', $mediaPath)->delete(); $this->createFileMeta($folder, $item); @@ -554,68 +558,52 @@ public function scan(MediaItem $folder = null, $path = null, $forceResync = fals /** * Creates a meta record for a folder in the "media_items" table, as a subfolder of the parent folder. - * - * @param MediaItem $parent - * @param array $meta - * @return MediaItem */ - protected function createFolderMeta(MediaItem $parent, array $meta) + protected function createFolderMeta(MediaItem $parent, DirectoryAttributes $meta): MediaItem { - try { - $path = self::validatePath($meta['path']); - } catch (ApplicationException $e) { - return; - } + $path = self::validatePath($meta->path()); return $parent->children()->create([ - 'name' => $meta['filename'], + 'name' => basename($path), 'path' => $this->getMediaRelativePath($path), 'type' => MediaLibraryItem::TYPE_FOLDER, 'size' => 0, - 'modified_at' => Argon::createFromTimestamp($meta['timestamp']), + 'modified_at' => Argon::createFromTimestamp($meta->lastModified()), ]); } /** * Creates a meta record for a file in the "media_items" table, as a child file of the parent folder. - * - * @param MediaItem $parent - * @param array $meta - * @return MediaItem */ - protected function createFileMeta(MediaItem $parent, array $meta) + protected function createFileMeta(MediaItem $parent, FileAttributes $meta): MediaItem { - try { - $path = self::validatePath($meta['path']); - } catch (ApplicationException $e) { - return; - } + $path = self::validatePath($meta->path()); $path = $this->getMediaRelativePath($path); // Create a temporary media library item instance $mediaItem = new MediaLibraryItem( $path, - $meta['size'], - $meta['timestamp'], + $meta->fileSize(), + $meta->lastModified(), MediaLibraryItem::TYPE_FILE, $this->getPathUrl($path) ); // Standard metadata $file = $parent->children()->make([ - 'name' => $meta['filename'], + 'name' => basename($path), 'path' => $path, 'type' => MediaLibraryItem::TYPE_FILE, - 'extension' => $meta['extension'], - 'size' => $meta['size'], - 'modified_at' => Argon::createFromTimestamp($meta['timestamp']), + 'extension' => strtolower(pathinfo($path, PATHINFO_EXTENSION)), + 'size' => $meta->fileSize(), + 'modified_at' => Argon::createFromTimestamp($meta->lastModified()), ]); // Extra metadata $file->file_type = $mediaItem->getFileType(); if ($mediaItem->getFileType() === MediaLibraryItem::FILE_TYPE_IMAGE) { - $this->setImageMeta($file, $meta['path']); + $this->setImageMeta($file, $meta->path()); } $file->save(); From f3ff8cc1b17ec0db45e3c7f37615c1997c700c01 Mon Sep 17 00:00:00 2001 From: Ben Thomson Date: Thu, 31 Aug 2023 15:39:50 +0800 Subject: [PATCH 14/14] Use new scope name --- modules/system/models/MediaItem.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/system/models/MediaItem.php b/modules/system/models/MediaItem.php index 43794bbdf1..ac2aeb4114 100644 --- a/modules/system/models/MediaItem.php +++ b/modules/system/models/MediaItem.php @@ -168,7 +168,7 @@ public function contents($sortBy = 'title', $filter = null, $ignoreFolders = fal */ public function folders($exclude = [], $includeSelf = true) { - $query = $this->allChildren()->where('type', MediaLibraryItem::TYPE_FOLDER); + $query = $this->descendants()->where('type', MediaLibraryItem::TYPE_FOLDER); $directories = $query->get(); @@ -205,7 +205,7 @@ public function search($searchTerm, $sortBy = 'title', $filter = null) // Normalise search term(s) $words = explode(' ', Str::lower($searchTerm)); - $query = $this->allChildren(); + $query = $this->descendants(); // Only return files as results $query->where('type', MediaLibraryItem::TYPE_FILE);