From 4ef3285eb20adef87afdea99fce791b1798bcee5 Mon Sep 17 00:00:00 2001 From: Andy Heathershaw Date: Sun, 21 Jul 2019 17:22:40 +0100 Subject: [PATCH] Fixes #123: Processing queue is now used for bulk uploads. If an error occurs during processing on the queue, this is now relayed to the user. Fixed an issue when creating an album where the default storage wasn't defaulted. --- app/Console/Commands/ProcessQueueCommand.php | 7 +- .../Controllers/Admin/PhotoController.php | 77 ++++++++++++------- ...add_error_columns_to_queue_items_table.php | 36 +++++++++ .../themes/base/admin/create_album.blade.php | 2 +- 4 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 database/migrations/2019_07_21_153126_add_error_columns_to_queue_items_table.php diff --git a/app/Console/Commands/ProcessQueueCommand.php b/app/Console/Commands/ProcessQueueCommand.php index e0ac46a..6c57d1a 100644 --- a/app/Console/Commands/ProcessQueueCommand.php +++ b/app/Console/Commands/ProcessQueueCommand.php @@ -11,7 +11,6 @@ use App\Services\RabbitMQService; use App\User; use App\UserActivity; use Illuminate\Console\Command; -use Illuminate\Support\Facades\App; class ProcessQueueCommand extends Command { @@ -109,9 +108,13 @@ class ProcessQueueCommand extends Command catch (\Exception $ex) { $this->output->error($ex->getMessage()); + $queueItem->error_message = $ex->getMessage(); } finally { + $queueItem->completed_at = new \DateTime(); + $queueItem->save(); + $msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']); } } @@ -155,6 +158,8 @@ class ProcessQueueCommand extends Command // Log an activity record for the user's feed $this->createActivityRecord($photo, 'photo.taken', $queueItem->user_id, $photo->taken_at); } + + $queueItem->is_successful = true; } private function processPhotoBulkActionMessage(QueueItem $queueItem) diff --git a/app/Http/Controllers/Admin/PhotoController.php b/app/Http/Controllers/Admin/PhotoController.php index 951b983..947e8e4 100644 --- a/app/Http/Controllers/Admin/PhotoController.php +++ b/app/Http/Controllers/Admin/PhotoController.php @@ -3,33 +3,25 @@ namespace App\Http\Controllers\Admin; use App\Album; -use App\AlbumSources\IAlbumSource; -use App\Facade\Image; -use App\Facade\Theme; use App\Facade\UserConfig; use App\Helpers\AnalysisQueueHelper; use App\Helpers\FileHelper; -use App\Helpers\ImageHelper; use App\Helpers\MiscHelper; +use App\Http\Controllers\Controller; use App\Http\Requests\UpdatePhotosBulkRequest; -use App\Label; use App\Photo; use App\QueueItem; use App\Services\PhotoService; use App\Services\RabbitMQService; -use App\Storage; use App\Upload; use App\UploadPhoto; -use App\User; use App\UserActivity; use Illuminate\Http\Request; -use App\Http\Controllers\Controller; use Illuminate\Http\UploadedFile; use Illuminate\Support\Facades\App; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\View; -use Symfony\Component\Finder\Iterator\RecursiveDirectoryIterator; use Symfony\Component\HttpFoundation\File\File; class PhotoController extends Controller @@ -81,11 +73,26 @@ class PhotoController extends Controller } } - $result['is_successful'] = !is_null($photoQueueItem->completed_at); - if (!$result['is_successful']) + $didComplete = !is_null($photoQueueItem->completed_at); + if (!$didComplete) { $result['message'] = 'Timed out waiting for queue processing.'; } + else if (!$photoQueueItem->is_successful) + { + $result['is_successful'] = false; + $result['message'] = $photoQueueItem->error_message; + + // Remove the photo from the album if it was newly-uploaded and couldn't be processed + if (intval($photo->metadata_version) === 0) + { + $photo->delete(); + } + } + else + { + $result['is_successful'] = true; + } } else { @@ -317,17 +324,16 @@ class PhotoController extends Controller $photo = Photo::where('id', intval($request->get('photo_id')))->first(); $photo->raw_exif_data = null; - /** @var File $savedFile */ - $queueFolder = FileHelper::getQueuePath($queueUid); - $savedFile = FileHelper::saveUploadedFile($photoFile, $queueFolder, $photo->storage_file_name); + $queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid, $photo->storage_file_name); + $uploadedTempFile = new File($photoFile); $this->removeExistingActivityRecords($photo, 'photo.uploaded'); $this->removeExistingActivityRecords($photo, 'photo.taken'); $photo->file_name = $photoFile->getClientOriginalName(); - $photo->mime_type = $savedFile->getMimeType(); - $photo->file_size = $savedFile->getSize(); - $photo->storage_file_name = $savedFile->getFilename(); + $photo->mime_type = $uploadedTempFile->getMimeType(); + $photo->file_size = $uploadedTempFile->getSize(); + $photo->storage_file_name = basename($queuedFileName); } else { @@ -413,7 +419,10 @@ class PhotoController extends Controller throw new \Exception('No queue_token value was provided!'); } - $queueFolder = FileHelper::getQueuePath($queueUid); + $temporaryFolder = sprintf('%s/%s', sys_get_temp_dir(), MiscHelper::randomString()); + @mkdir($temporaryFolder); + + $queueStorage = AnalysisQueueHelper::getStorageQueueSource(); $mimeType = strtolower($archiveFile->getMimeType()); switch ($mimeType) @@ -421,7 +430,7 @@ class PhotoController extends Controller case 'application/zip': $zip = new \ZipArchive(); $zip->open($archiveFile->getPathname()); - $zip->extractTo($queueFolder); + $zip->extractTo($temporaryFolder); $zip->close(); @unlink($archiveFile->getPathname()); break; @@ -431,7 +440,7 @@ class PhotoController extends Controller return redirect(route('albums.show', ['id' => $album->id])); } - $di = new \RecursiveDirectoryIterator($queueFolder, \RecursiveDirectoryIterator::SKIP_DOTS); + $di = new \RecursiveDirectoryIterator($temporaryFolder, \RecursiveDirectoryIterator::SKIP_DOTS); $recursive = new \RecursiveIteratorIterator($di); /** @var \SplFileInfo $fileInfo */ @@ -463,24 +472,38 @@ class PhotoController extends Controller } $photoFile = new File($fileInfo->getPathname()); - - /** @var File $savedFile */ - $savedFile = FileHelper::saveExtractedFile($photoFile, $queueFolder); + $queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid); $photo = new Photo(); $photo->album_id = $album->id; $photo->user_id = Auth::user()->id; $photo->name = pathinfo($photoFile->getFilename(), PATHINFO_FILENAME); $photo->file_name = $photoFile->getFilename(); - $photo->storage_file_name = $savedFile->getFilename(); - $photo->mime_type = $savedFile->getMimeType(); - $photo->file_size = $savedFile->getSize(); + $photo->mime_type = $photoFile->getMimeType(); + $photo->file_size = $photoFile->getSize(); $photo->is_analysed = false; + $photo->storage_file_name = basename($queuedFileName); $photo->save(); - // Log an activity record for the user's feed // Log an activity record for the user's feed $this->createActivityRecord($photo, 'photo.uploaded'); + + // If queueing is enabled, store the photo in the queue now + if (UserConfig::isImageProcessingQueueEnabled()) + { + $queueItem = new QueueItem([ + 'batch_reference' => $queueUid, + 'action_type' => 'photo.analyse', + 'album_id' => $photo->album_id, + 'photo_id' => $photo->id, + 'user_id' => $this->getUser()->id, + 'queued_at' => new \DateTime() + ]); + $queueItem->save(); + + $rabbitmq = new RabbitMQService(); + $rabbitmq->queueItem($queueItem); + } } return redirect(route('albums.analyse', [ diff --git a/database/migrations/2019_07_21_153126_add_error_columns_to_queue_items_table.php b/database/migrations/2019_07_21_153126_add_error_columns_to_queue_items_table.php new file mode 100644 index 0000000..3ba0c65 --- /dev/null +++ b/database/migrations/2019_07_21_153126_add_error_columns_to_queue_items_table.php @@ -0,0 +1,36 @@ +boolean('is_successful')->default(false); + $table->text('error_message')->nullable(true); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('queue_items', function (Blueprint $table) + { + $table->dropColumn('is_successful'); + $table->dropColumn('error_message'); + }); + } +} diff --git a/resources/views/themes/base/admin/create_album.blade.php b/resources/views/themes/base/admin/create_album.blade.php index 0f41cce..e9b1f1d 100644 --- a/resources/views/themes/base/admin/create_album.blade.php +++ b/resources/views/themes/base/admin/create_album.blade.php @@ -49,7 +49,7 @@