Fixes #123: Processing queue is now used for bulk uploads. If an error occurs during processing on the queue, this is now relayed to the user. Fixed an issue when creating an album where the default storage wasn't defaulted.

This commit is contained in:
Andy Heathershaw 2019-07-21 17:22:40 +01:00
parent 3b76f20738
commit 4ef3285eb2
4 changed files with 93 additions and 29 deletions

View File

@ -11,7 +11,6 @@ use App\Services\RabbitMQService;
use App\User;
use App\UserActivity;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\App;
class ProcessQueueCommand extends Command
{
@ -109,9 +108,13 @@ class ProcessQueueCommand extends Command
catch (\Exception $ex)
{
$this->output->error($ex->getMessage());
$queueItem->error_message = $ex->getMessage();
}
finally
{
$queueItem->completed_at = new \DateTime();
$queueItem->save();
$msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
}
}
@ -155,6 +158,8 @@ class ProcessQueueCommand extends Command
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.taken', $queueItem->user_id, $photo->taken_at);
}
$queueItem->is_successful = true;
}
private function processPhotoBulkActionMessage(QueueItem $queueItem)

View File

@ -3,33 +3,25 @@
namespace App\Http\Controllers\Admin;
use App\Album;
use App\AlbumSources\IAlbumSource;
use App\Facade\Image;
use App\Facade\Theme;
use App\Facade\UserConfig;
use App\Helpers\AnalysisQueueHelper;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper;
use App\Helpers\MiscHelper;
use App\Http\Controllers\Controller;
use App\Http\Requests\UpdatePhotosBulkRequest;
use App\Label;
use App\Photo;
use App\QueueItem;
use App\Services\PhotoService;
use App\Services\RabbitMQService;
use App\Storage;
use App\Upload;
use App\UploadPhoto;
use App\User;
use App\UserActivity;
use Illuminate\Http\Request;
use App\Http\Controllers\Controller;
use Illuminate\Http\UploadedFile;
use Illuminate\Support\Facades\App;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\View;
use Symfony\Component\Finder\Iterator\RecursiveDirectoryIterator;
use Symfony\Component\HttpFoundation\File\File;
class PhotoController extends Controller
@ -81,11 +73,26 @@ class PhotoController extends Controller
}
}
$result['is_successful'] = !is_null($photoQueueItem->completed_at);
if (!$result['is_successful'])
$didComplete = !is_null($photoQueueItem->completed_at);
if (!$didComplete)
{
$result['message'] = 'Timed out waiting for queue processing.';
}
else if (!$photoQueueItem->is_successful)
{
$result['is_successful'] = false;
$result['message'] = $photoQueueItem->error_message;
// Remove the photo from the album if it was newly-uploaded and couldn't be processed
if (intval($photo->metadata_version) === 0)
{
$photo->delete();
}
}
else
{
$result['is_successful'] = true;
}
}
else
{
@ -317,17 +324,16 @@ class PhotoController extends Controller
$photo = Photo::where('id', intval($request->get('photo_id')))->first();
$photo->raw_exif_data = null;
/** @var File $savedFile */
$queueFolder = FileHelper::getQueuePath($queueUid);
$savedFile = FileHelper::saveUploadedFile($photoFile, $queueFolder, $photo->storage_file_name);
$queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid, $photo->storage_file_name);
$uploadedTempFile = new File($photoFile);
$this->removeExistingActivityRecords($photo, 'photo.uploaded');
$this->removeExistingActivityRecords($photo, 'photo.taken');
$photo->file_name = $photoFile->getClientOriginalName();
$photo->mime_type = $savedFile->getMimeType();
$photo->file_size = $savedFile->getSize();
$photo->storage_file_name = $savedFile->getFilename();
$photo->mime_type = $uploadedTempFile->getMimeType();
$photo->file_size = $uploadedTempFile->getSize();
$photo->storage_file_name = basename($queuedFileName);
}
else
{
@ -413,7 +419,10 @@ class PhotoController extends Controller
throw new \Exception('No queue_token value was provided!');
}
$queueFolder = FileHelper::getQueuePath($queueUid);
$temporaryFolder = sprintf('%s/%s', sys_get_temp_dir(), MiscHelper::randomString());
@mkdir($temporaryFolder);
$queueStorage = AnalysisQueueHelper::getStorageQueueSource();
$mimeType = strtolower($archiveFile->getMimeType());
switch ($mimeType)
@ -421,7 +430,7 @@ class PhotoController extends Controller
case 'application/zip':
$zip = new \ZipArchive();
$zip->open($archiveFile->getPathname());
$zip->extractTo($queueFolder);
$zip->extractTo($temporaryFolder);
$zip->close();
@unlink($archiveFile->getPathname());
break;
@ -431,7 +440,7 @@ class PhotoController extends Controller
return redirect(route('albums.show', ['id' => $album->id]));
}
$di = new \RecursiveDirectoryIterator($queueFolder, \RecursiveDirectoryIterator::SKIP_DOTS);
$di = new \RecursiveDirectoryIterator($temporaryFolder, \RecursiveDirectoryIterator::SKIP_DOTS);
$recursive = new \RecursiveIteratorIterator($di);
/** @var \SplFileInfo $fileInfo */
@ -463,24 +472,38 @@ class PhotoController extends Controller
}
$photoFile = new File($fileInfo->getPathname());
/** @var File $savedFile */
$savedFile = FileHelper::saveExtractedFile($photoFile, $queueFolder);
$queuedFileName = $queueStorage->uploadToAnalysisQueue($photoFile, $queueUid);
$photo = new Photo();
$photo->album_id = $album->id;
$photo->user_id = Auth::user()->id;
$photo->name = pathinfo($photoFile->getFilename(), PATHINFO_FILENAME);
$photo->file_name = $photoFile->getFilename();
$photo->storage_file_name = $savedFile->getFilename();
$photo->mime_type = $savedFile->getMimeType();
$photo->file_size = $savedFile->getSize();
$photo->mime_type = $photoFile->getMimeType();
$photo->file_size = $photoFile->getSize();
$photo->is_analysed = false;
$photo->storage_file_name = basename($queuedFileName);
$photo->save();
// Log an activity record for the user's feed
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.uploaded');
// If queueing is enabled, store the photo in the queue now
if (UserConfig::isImageProcessingQueueEnabled())
{
$queueItem = new QueueItem([
'batch_reference' => $queueUid,
'action_type' => 'photo.analyse',
'album_id' => $photo->album_id,
'photo_id' => $photo->id,
'user_id' => $this->getUser()->id,
'queued_at' => new \DateTime()
]);
$queueItem->save();
$rabbitmq = new RabbitMQService();
$rabbitmq->queueItem($queueItem);
}
}
return redirect(route('albums.analyse', [

View File

@ -0,0 +1,36 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class AddErrorColumnsToQueueItemsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('queue_items', function (Blueprint $table)
{
$table->boolean('is_successful')->default(false);
$table->text('error_message')->nullable(true);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('queue_items', function (Blueprint $table)
{
$table->dropColumn('is_successful');
$table->dropColumn('error_message');
});
}
}

View File

@ -49,7 +49,7 @@
<label class="form-control-label" for="album-source">@lang('forms.album_source_label')</label>
<select class="form-control" name="storage_id" id="album-source">
@foreach ($album_sources as $key => $value)
<option value="{{ $key }}"{{ $key == old('storage_id') ? ' selected="selected"' : '' }}>{{ $value }}</option>
<option value="{{ $key }}"{{ $key == old('storage_id', $default_storage_id) ? ' selected="selected"' : '' }}>{{ $value }}</option>
@endforeach
</select>
</div>