From 95e79f2d28eaf545c31b4429bc16628e4b8610fc Mon Sep 17 00:00:00 2001 From: Andy Heathershaw Date: Sat, 13 Jul 2019 10:15:13 +0100 Subject: [PATCH] #123: The process command (which is now bt-queue:process to avoid conflicts with Laravel's default queue namespace) now uses the S3 storage to retrieve images --- app/AlbumSources/AmazonS3Source.php | 48 +++++++++++++++++++ app/AlbumSources/IAnalysisQueueSource.php | 16 +++++++ app/Console/Commands/ProcessQueueCommand.php | 17 +++---- .../Controllers/Admin/PhotoController.php | 12 +++++ app/Services/PhotoService.php | 13 +++-- ...9_07_13_075909_add_new_album_id_column.php | 38 +++++++++++++++ resources/systemd/blue-twilight-mq.service | 2 +- 7 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 database/migrations/2019_07_13_075909_add_new_album_id_column.php diff --git a/app/AlbumSources/AmazonS3Source.php b/app/AlbumSources/AmazonS3Source.php index 58fe668..6694f5b 100644 --- a/app/AlbumSources/AmazonS3Source.php +++ b/app/AlbumSources/AmazonS3Source.php @@ -20,6 +20,30 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ // The delete routine will have already removed all photos } + /** + * Deletes a photo to be analysed from the storage source + * @param string $queueToken Queue token holding the photo + * @param string $fileName Filename of the photo to download + * @return void + */ + public function deleteItemFromAnalysisQueue($queueToken, $fileName) + { + $fileToDelete = $this->getPathToAnalysisQueueItem($queueToken, $fileName); + + try + { + $this->getClient()->deleteObject([ + 'Bucket' => $this->configuration->container_name, + 'Key' => $fileToDelete + ]); + } + catch (ClientErrorResponseException $ex) + { + // Don't worry if the file no longer exists + Log::warning('Failed deleting image from S3.', ['error' => $ex->getMessage(), 'path' => $fileToDelete]); + } + } + /** * Deletes a thumbnail file for a photo. * @param Photo $photo Photo to delete the thumbnail from. @@ -44,6 +68,25 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ } } + /** + * Downloads a photo to be analysed from the storage source to a temporary file + * @param string $queueToken Queue token holding the photo + * @param string $fileName Filename of the photo to download + * @return string Path to the photo that was downloaded + */ + public function fetchItemFromAnalysisQueue($queueToken, $fileName) + { + $tempFile = tempnam(sys_get_temp_dir(), 'BlueTwilight_'); + + $this->getClient()->getObject([ + 'Bucket' => $this->configuration->container_name, + 'Key' => $this->getPathToAnalysisQueueItem($queueToken, $fileName), + 'SaveAs' => $tempFile + ]); + + return $tempFile; + } + /** * Fetches the contents of a thumbnail for a photo. * @param Photo $photo Photo to fetch the thumbnail for. @@ -155,6 +198,11 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ return sprintf('analysis-queue/%s', $queueToken); } + private function getPathToAnalysisQueueItem($queueToken, $fileName) + { + return sprintf('%s/%s', $this->getPathToAnalysisQueue($queueToken), $fileName); + } + private function getPathToPhoto(Photo $photo, $thumbnail = null) { return sprintf( diff --git a/app/AlbumSources/IAnalysisQueueSource.php b/app/AlbumSources/IAnalysisQueueSource.php index b5ec3b5..ea2ba6f 100644 --- a/app/AlbumSources/IAnalysisQueueSource.php +++ b/app/AlbumSources/IAnalysisQueueSource.php @@ -6,6 +6,22 @@ use App\Storage; interface IAnalysisQueueSource { + /** + * Deletes a photo to be analysed from the storage source + * @param string $queueToken Queue token holding the photo + * @param string $fileName Filename of the photo to download + * @return void + */ + function deleteItemFromAnalysisQueue($queueToken, $fileName); + + /** + * Downloads a photo to be analysed from the storage source to a temporary file + * @param string $queueToken Queue token holding the photo + * @param string $fileName Filename of the photo to download + * @return string Path to the photo that was downloaded + */ + function fetchItemFromAnalysisQueue($queueToken, $fileName); + /** * @param Storage $configuration * @return mixed diff --git a/app/Console/Commands/ProcessQueueCommand.php b/app/Console/Commands/ProcessQueueCommand.php index 0ed8c07..e0ac46a 100644 --- a/app/Console/Commands/ProcessQueueCommand.php +++ b/app/Console/Commands/ProcessQueueCommand.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Album; use App\Facade\UserConfig; use App\Photo; use App\QueueItem; @@ -10,6 +11,7 @@ use App\Services\RabbitMQService; use App\User; use App\UserActivity; use Illuminate\Console\Command; +use Illuminate\Support\Facades\App; class ProcessQueueCommand extends Command { @@ -18,7 +20,7 @@ class ProcessQueueCommand extends Command * * @var string */ - protected $signature = 'queue:process'; + protected $signature = 'bt-queue:process'; /** * The console command description. @@ -181,21 +183,20 @@ class ProcessQueueCommand extends Command switch (strtolower($action)) { /* This needs a bit more work - we need to also store the new album ID in the queue_items table */ - /*case 'change_album': + case 'change_album': if ($user->can('change-metadata', $photo)) { - $newAlbumId = intval($request->get('new-album-id')); - if ($newAlbumId == $photo->album_id) + $newAlbum = Album::where('id', intval($queueItem->new_album_id))->first(); + if (is_null($newAlbum) || !$user->can('upload-photos', $newAlbum)) { - // Photo already belongs to this album, don't move + $this->output->writeln('Target album does not exist or user does not have permission.'); return; } - $newAlbum = $this->loadAlbum($newAlbumId, 'upload-photos'); + $this->output->writeln(sprintf('Moving photo to album \'%s\'', $newAlbum->name)); $photoService->changeAlbum($newAlbum); - $changed = true; } - break;*/ + break; case 'delete': if ($user->can('delete', $photo)) diff --git a/app/Http/Controllers/Admin/PhotoController.php b/app/Http/Controllers/Admin/PhotoController.php index 6579636..951b983 100644 --- a/app/Http/Controllers/Admin/PhotoController.php +++ b/app/Http/Controllers/Admin/PhotoController.php @@ -601,6 +601,18 @@ class PhotoController extends Controller 'user_id' => $this->getUser()->id, 'queued_at' => new \DateTime() ]); + + if (strtolower($action) == 'change_album') + { + $queueItem->new_album_id = intval($request->get('new-album-id')); + $newAlbumId = intval($request->get('new-album-id')); + if ($newAlbumId == $photo->album_id) + { + // Photo already belongs to this album, don't move + continue; + } + } + $queueItem->save(); $rabbitmq = new RabbitMQService(); diff --git a/app/Services/PhotoService.php b/app/Services/PhotoService.php index ae8ba94..59055fd 100644 --- a/app/Services/PhotoService.php +++ b/app/Services/PhotoService.php @@ -4,6 +4,8 @@ namespace App\Services; use App\Album; use App\AlbumSources\IAlbumSource; +use App\AlbumSources\IAnalysisQueueSource; +use App\Helpers\AnalysisQueueHelper; use App\Helpers\FileHelper; use App\Helpers\ImageHelper; use App\Helpers\MiscHelper; @@ -52,8 +54,10 @@ class PhotoService public function analyse($queueToken, $isReanalyse = false) { - $queuePath = FileHelper::getQueuePath($queueToken); - $photoFile = join(DIRECTORY_SEPARATOR, [$queuePath, $this->photo->storage_file_name]); + /** @var IAnalysisQueueSource $analysisQueueStorage */ + $analysisQueueStorage = AnalysisQueueHelper::getStorageQueueSource(); + + $photoFile = $analysisQueueStorage->fetchItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name); try { @@ -143,10 +147,11 @@ class PhotoService } finally { + // Remove the temporary file @unlink($photoFile); - // If the queue directory is now empty, get rid of it - FileHelper::deleteIfEmpty($queuePath); + // Remove from the storage + $analysisQueueStorage->deleteItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name); } } diff --git a/database/migrations/2019_07_13_075909_add_new_album_id_column.php b/database/migrations/2019_07_13_075909_add_new_album_id_column.php new file mode 100644 index 0000000..4cd1cfe --- /dev/null +++ b/database/migrations/2019_07_13_075909_add_new_album_id_column.php @@ -0,0 +1,38 @@ +unsignedInteger('new_album_id')->nullable(true); + + $table->foreign('new_album_id') + ->references('id')->on('albums') + ->onDelete('cascade'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('user_activity', function (Blueprint $table) + { + $table->dropColumn('new_album_id'); + }); + } +} diff --git a/resources/systemd/blue-twilight-mq.service b/resources/systemd/blue-twilight-mq.service index f727d14..d61fc18 100644 --- a/resources/systemd/blue-twilight-mq.service +++ b/resources/systemd/blue-twilight-mq.service @@ -3,7 +3,7 @@ Description=Blue Twilight Processing Queue Runner [Service] WorkingDirectory=/data/www/blue-twilight.andysh.dev/ -ExecStart=/usr/bin/php artisan queue:process +ExecStart=/usr/bin/php artisan bt-queue:process Restart=on-failure User=www-data Group=www-data