Fixes #123: Processing queue is now used for bulk uploads. If an error occurs during processing on the queue, this is now relayed to the user. Fixed an issue when creating an album where the default storage wasn't defaulted.

app/Console/Commands/ProcessQueueCommand.php

@@ -11,7 +11,6 @@ use App\Services\RabbitMQService;
 use App\User;
 use App\UserActivity;
 use Illuminate\Console\Command;
-use Illuminate\Support\Facades\App;
 
 class ProcessQueueCommand extends Command
 {
@@ -109,9 +108,13 @@ class ProcessQueueCommand extends Command
         catch (\Exception $ex)
         {
             $this->output->error($ex->getMessage());
+            $queueItem->error_message = $ex->getMessage();
         }
         finally
         {
+            $queueItem->completed_at = new \DateTime();
+            $queueItem->save();
+
             $msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
         }
     }
@@ -155,6 +158,8 @@ class ProcessQueueCommand extends Command
             // Log an activity record for the user's feed
             $this->createActivityRecord($photo, 'photo.taken', $queueItem->user_id, $photo->taken_at);
         }
+
+        $queueItem->is_successful = true;
     }
 
     private function processPhotoBulkActionMessage(QueueItem $queueItem)

app/Http/Controllers/Admin/PhotoController.php

@@ -3,33 +3,25 @@
 namespace App\Http\Controllers\Admin;
 
 use App\Album;
-use App\AlbumSources\IAlbumSource;
-use App\Facade\Image;
-use App\Facade\Theme;
 use App\Facade\UserConfig;
 use App\Helpers\AnalysisQueueHelper;
 use App\Helpers\FileHelper;
-use App\Helpers\ImageHelper;
 use App\Helpers\MiscHelper;
+use App\Http\Controllers\Controller;
 use App\Http\Requests\UpdatePhotosBulkRequest;
-use App\Label;
 use App\Photo;
 use App\QueueItem;
 use App\Services\PhotoService;
 use App\Services\RabbitMQService;
-use App\Storage;
 use App\Upload;
 use App\UploadPhoto;
-use App\User;
 use App\UserActivity;
 use Illuminate\Http\Request;
-use App\Http\Controllers\Controller;
 use Illuminate\Http\UploadedFile;
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\View;
-use Symfony\Component\Finder\Iterator\RecursiveDirectoryIterator;
 use Symfony\Component\HttpFoundation\File\File;
 
 class PhotoController extends Controller
@@ -81,11 +73,26 @@ class PhotoController extends Controller
                     }
                 }
 
-                $result['is_successful'] = !is_null($photoQueueItem->completed_at);
+                $didComplete = !is_null($photoQueueItem->completed_at);
-                if (!$result['is_successful'])
+                if (!$didComplete)
                 {
                     $result['message'] = 'Timed out waiting for queue processing.';
                 }
+                else if (!$photoQueueItem->is_successful)
+                {
+                    $result['is_successful'] = false;
+                    $result['message'] = $photoQueueItem->error_message;
+
+                    // Remove the photo from the album if it was newly-uploaded and couldn't be processed
+                    if (intval($photo->metadata_version) === 0)
+                    {
+                        $photo->delete();
+                    }
+                }
+                else
+                {
+                    $result['is_successful'] = true;
+                }
             }
             else
             {
@@ -317,17 +324,16 @@ class PhotoController extends Controller
                     $photo = Photo::where('id', intval($request->get('photo_id')))->first();
                     $photo->raw_exif_data = null;
 
-                    /** @var File $savedFile */
+                    $queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid, $photo->storage_file_name);
-                    $queueFolder = FileHelper::getQueuePath($queueUid);
+                    $uploadedTempFile = new File($photoFile);
-                    $savedFile = FileHelper::saveUploadedFile($photoFile, $queueFolder, $photo->storage_file_name);
 
                     $this->removeExistingActivityRecords($photo, 'photo.uploaded');
                     $this->removeExistingActivityRecords($photo, 'photo.taken');
 
                     $photo->file_name = $photoFile->getClientOriginalName();
-                    $photo->mime_type = $savedFile->getMimeType();
+                    $photo->mime_type = $uploadedTempFile->getMimeType();
-                    $photo->file_size = $savedFile->getSize();
+                    $photo->file_size = $uploadedTempFile->getSize();
-                    $photo->storage_file_name = $savedFile->getFilename();
+                    $photo->storage_file_name = basename($queuedFileName);
                 }
                 else
                 {
@@ -413,7 +419,10 @@ class PhotoController extends Controller
             throw new \Exception('No queue_token value was provided!');
         }
 
-        $queueFolder = FileHelper::getQueuePath($queueUid);
+        $temporaryFolder = sprintf('%s/%s', sys_get_temp_dir(), MiscHelper::randomString());
+        @mkdir($temporaryFolder);
+
+        $queueStorage = AnalysisQueueHelper::getStorageQueueSource();
 
         $mimeType = strtolower($archiveFile->getMimeType());
         switch ($mimeType)
@@ -421,7 +430,7 @@ class PhotoController extends Controller
             case 'application/zip':
                 $zip = new \ZipArchive();
                 $zip->open($archiveFile->getPathname());
-                $zip->extractTo($queueFolder);
+                $zip->extractTo($temporaryFolder);
                 $zip->close();
                 @unlink($archiveFile->getPathname());
                 break;
@@ -431,7 +440,7 @@ class PhotoController extends Controller
                 return redirect(route('albums.show', ['id' => $album->id]));
         }
 
-        $di = new \RecursiveDirectoryIterator($queueFolder, \RecursiveDirectoryIterator::SKIP_DOTS);
+        $di = new \RecursiveDirectoryIterator($temporaryFolder, \RecursiveDirectoryIterator::SKIP_DOTS);
         $recursive = new \RecursiveIteratorIterator($di);
 
         /** @var \SplFileInfo $fileInfo */
@@ -463,24 +472,38 @@ class PhotoController extends Controller
             }
 
             $photoFile = new File($fileInfo->getPathname());
-
+            $queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid);
-            /** @var File $savedFile */
-            $savedFile = FileHelper::saveExtractedFile($photoFile, $queueFolder);
 
             $photo = new Photo();
             $photo->album_id = $album->id;
             $photo->user_id = Auth::user()->id;
             $photo->name = pathinfo($photoFile->getFilename(), PATHINFO_FILENAME);
             $photo->file_name = $photoFile->getFilename();
-            $photo->storage_file_name = $savedFile->getFilename();
+            $photo->mime_type = $photoFile->getMimeType();
-            $photo->mime_type = $savedFile->getMimeType();
+            $photo->file_size = $photoFile->getSize();
-            $photo->file_size = $savedFile->getSize();
             $photo->is_analysed = false;
+            $photo->storage_file_name = basename($queuedFileName);
             $photo->save();
 
-            // Log an activity record for the user's feed
             // Log an activity record for the user's feed
             $this->createActivityRecord($photo, 'photo.uploaded');
+
+            // If queueing is enabled, store the photo in the queue now
+            if (UserConfig::isImageProcessingQueueEnabled())
+            {
+                $queueItem = new QueueItem([
+                    'batch_reference' => $queueUid,
+                    'action_type' => 'photo.analyse',
+                    'album_id' => $photo->album_id,
+                    'photo_id' => $photo->id,
+                    'user_id' => $this->getUser()->id,
+                    'queued_at' => new \DateTime()
+                ]);
+                $queueItem->save();
+
+                $rabbitmq = new RabbitMQService();
+                $rabbitmq->queueItem($queueItem);
+            }
         }
 
         return redirect(route('albums.analyse', [

database/migrations/2019_07_21_153126_add_error_columns_to_queue_items_table.php

@@ -0,0 +1,36 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddErrorColumnsToQueueItemsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('queue_items', function (Blueprint $table)
+        {
+            $table->boolean('is_successful')->default(false);
+            $table->text('error_message')->nullable(true);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('queue_items', function (Blueprint $table)
+        {
+            $table->dropColumn('is_successful');
+            $table->dropColumn('error_message');
+        });
+    }
+}

resources/views/themes/base/admin/create_album.blade.php

@@ -49,7 +49,7 @@
                         <label class="form-control-label" for="album-source">@lang('forms.album_source_label')</label>
                         <select class="form-control" name="storage_id" id="album-source">
                             @foreach ($album_sources as $key => $value)
-                                <option value="{{ $key }}"{{ $key == old('storage_id') ? ' selected="selected"' : '' }}>{{ $value }}</option>
+                                <option value="{{ $key }}"{{ $key == old('storage_id', $default_storage_id) ? ' selected="selected"' : '' }}>{{ $value }}</option>
                             @endforeach
                         </select>
                     </div>