aheathershaw
/
blue-twilight
1
0
Fork 0
Code Issues 32 Pull Requests Releases 15 Wiki Activity

#123: The process command (which is now bt-queue:process to avoid conflicts with Laravel's default queue namespace) now uses the S3 storage to retrieve images

This commit is contained in:
Andy Heathershaw 2019-07-13 10:15:13 +01:00
parent 7a71a06e4e
commit 95e79f2d28
7 changed files with 133 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
app/AlbumSources/AmazonS3Source.php
View File

@ -20,6 +20,30 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
// The delete routine will have already removed all photos
}
/**
* Deletes a photo to be analysed from the storage source
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return void
*/
public function deleteItemFromAnalysisQueue($queueToken, $fileName)
{
$fileToDelete = $this->getPathToAnalysisQueueItem($queueToken, $fileName);
try
{
$this->getClient()->deleteObject([
'Bucket' => $this->configuration->container_name,
'Key' => $fileToDelete
]);
}
catch (ClientErrorResponseException $ex)
{
// Don't worry if the file no longer exists
Log::warning('Failed deleting image from S3.', ['error' => $ex->getMessage(), 'path' => $fileToDelete]);
}
}
/**
* Deletes a thumbnail file for a photo.
* @param Photo $photo Photo to delete the thumbnail from.
@ -44,6 +68,25 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
}
}
/**
* Downloads a photo to be analysed from the storage source to a temporary file
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return string Path to the photo that was downloaded
*/
public function fetchItemFromAnalysisQueue($queueToken, $fileName)
{
$tempFile = tempnam(sys_get_temp_dir(), 'BlueTwilight_');
$this->getClient()->getObject([
'Bucket' => $this->configuration->container_name,
'Key' => $this->getPathToAnalysisQueueItem($queueToken, $fileName),
'SaveAs' => $tempFile
]);
return $tempFile;
}
/**
* Fetches the contents of a thumbnail for a photo.
* @param Photo $photo Photo to fetch the thumbnail for.
@ -155,6 +198,11 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
return sprintf('analysis-queue/%s', $queueToken);
}
private function getPathToAnalysisQueueItem($queueToken, $fileName)
{
return sprintf('%s/%s', $this->getPathToAnalysisQueue($queueToken), $fileName);
}
private function getPathToPhoto(Photo $photo, $thumbnail = null)
{
return sprintf(

16
app/AlbumSources/IAnalysisQueueSource.php
View File

@ -6,6 +6,22 @@ use App\Storage;
interface IAnalysisQueueSource
{
/**
* Deletes a photo to be analysed from the storage source
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return void
*/
function deleteItemFromAnalysisQueue($queueToken, $fileName);
/**
* Downloads a photo to be analysed from the storage source to a temporary file
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return string Path to the photo that was downloaded
*/
function fetchItemFromAnalysisQueue($queueToken, $fileName);
/**
* @param Storage $configuration
* @return mixed

17
app/Console/Commands/ProcessQueueCommand.php
View File

@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Album;
use App\Facade\UserConfig;
use App\Photo;
use App\QueueItem;
@ -10,6 +11,7 @@ use App\Services\RabbitMQService;
use App\User;
use App\UserActivity;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\App;
class ProcessQueueCommand extends Command
{
@ -18,7 +20,7 @@ class ProcessQueueCommand extends Command
*
* @var string
*/
protected $signature = 'queue:process';
protected $signature = 'bt-queue:process';
/**
* The console command description.
@ -181,21 +183,20 @@ class ProcessQueueCommand extends Command
switch (strtolower($action))
{
/* This needs a bit more work - we need to also store the new album ID in the queue_items table */
/*case 'change_album':
case 'change_album':
if ($user->can('change-metadata', $photo))
{
$newAlbumId = intval($request->get('new-album-id'));
if ($newAlbumId == $photo->album_id)
$newAlbum = Album::where('id', intval($queueItem->new_album_id))->first();
if (is_null($newAlbum) || !$user->can('upload-photos', $newAlbum))
{
// Photo already belongs to this album, don't move
$this->output->writeln('Target album does not exist or user does not have permission.');
return;
}
$newAlbum = $this->loadAlbum($newAlbumId, 'upload-photos');
$this->output->writeln(sprintf('Moving photo to album \'%s\'', $newAlbum->name));
$photoService->changeAlbum($newAlbum);
$changed = true;
}
break;*/
break;
case 'delete':
if ($user->can('delete', $photo))

12
app/Http/Controllers/Admin/PhotoController.php
View File

@ -601,6 +601,18 @@ class PhotoController extends Controller
'user_id' => $this->getUser()->id,
'queued_at' => new \DateTime()
]);
if (strtolower($action) == 'change_album')
{
$queueItem->new_album_id = intval($request->get('new-album-id'));
$newAlbumId = intval($request->get('new-album-id'));
if ($newAlbumId == $photo->album_id)
{
// Photo already belongs to this album, don't move
continue;
}
}
$queueItem->save();
$rabbitmq = new RabbitMQService();

13
app/Services/PhotoService.php
View File

@ -4,6 +4,8 @@ namespace App\Services;
use App\Album;
use App\AlbumSources\IAlbumSource;
use App\AlbumSources\IAnalysisQueueSource;
use App\Helpers\AnalysisQueueHelper;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper;
use App\Helpers\MiscHelper;
@ -52,8 +54,10 @@ class PhotoService
public function analyse($queueToken, $isReanalyse = false)
{
$queuePath = FileHelper::getQueuePath($queueToken);
$photoFile = join(DIRECTORY_SEPARATOR, [$queuePath, $this->photo->storage_file_name]);
/** @var IAnalysisQueueSource $analysisQueueStorage */
$analysisQueueStorage = AnalysisQueueHelper::getStorageQueueSource();
$photoFile = $analysisQueueStorage->fetchItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name);
try
{
@ -143,10 +147,11 @@ class PhotoService
}
finally
{
// Remove the temporary file
@unlink($photoFile);
// If the queue directory is now empty, get rid of it
FileHelper::deleteIfEmpty($queuePath);
// Remove from the storage
$analysisQueueStorage->deleteItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name);
}
}

38
database/migrations/2019_07_13_075909_add_new_album_id_column.php Normal file
View File

@ -0,0 +1,38 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class AddNewAlbumIdColumn extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('user_activity', function (Blueprint $table)
{
$table->unsignedInteger('new_album_id')->nullable(true);
$table->foreign('new_album_id')
->references('id')->on('albums')
->onDelete('cascade');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('user_activity', function (Blueprint $table)
{
$table->dropColumn('new_album_id');
});
}
}

2
resources/systemd/blue-twilight-mq.service
View File

@ -3,7 +3,7 @@ Description=Blue Twilight Processing Queue Runner
[Service]
WorkingDirectory=/data/www/blue-twilight.andysh.dev/
ExecStart=/usr/bin/php artisan queue:process
ExecStart=/usr/bin/php artisan bt-queue:process
Restart=on-failure
User=www-data
Group=www-data