aheathershaw
/
blue-twilight
1
0
Fork 0
Code Issues 32 Pull Requests Releases 15 Wiki Activity

Merge branch 'feature/121-rabbitmq-queuing' of aheathershaw/blue-twilight into master

This commit is contained in:
Andy Heathershaw 2019-07-09 23:07:27 +01:00 committed by Gitea
parent dbdf17cc9e 0de33065fe
commit 3995d79955
12 changed files with 1231 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
app/Console/Commands/ProcessQueueCommand.php Normal file
View File

@ -0,0 +1,158 @@
<?php
namespace App\Console\Commands;
use App\Facade\UserConfig;
use App\Photo;
use App\QueueItem;
use App\Services\PhotoService;
use App\Services\RabbitMQService;
use App\UserActivity;
use Illuminate\Console\Command;
class ProcessQueueCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'queue:process';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Processes items in the processing queue.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
if (!UserConfig::isImageProcessingQueueEnabled())
{
$this->output->error('The image processing queue is not enabled');
}
$rabbitmq = new RabbitMQService();
$this->output->writeln('Monitoring queue');
$rabbitmq->waitOnQueue([$this, 'processQueueItem']);
}
/**
* Processes a single item from the queue.
*
* @param $msg
* @return void
*/
public function processQueueItem($msg)
{
$queueItemID = intval($msg->body);
$this->output->writeln(sprintf('Processing queue item %d', $queueItemID));
/** @var QueueItem $queueItem */
$queueItem = QueueItem::where('id', $queueItemID)->first();
if (is_null($queueItem))
{
$this->output->writeln('Queue item does not exist; skipping');
$msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
return;
}
try
{
switch (strtolower($queueItem->action_type))
{
case 'photo.analyse':
$this->processPhotoAnalyseMessage($queueItem);
break;
default:
$this->output->writeln(sprintf('Action %s is not recognised, skipping', $queueItem->action_type));
return;
}
$queueItem->completed_at = new \DateTime();
$queueItem->save();
}
catch (\Exception $ex)
{
$this->output->error($ex->getMessage());
}
finally
{
$msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
}
}
private function createActivityRecord(Photo $photo, $type, $activityDateTime = null)
{
if (is_null($activityDateTime))
{
$activityDateTime = new \DateTime();
}
$userActivity = new UserActivity();
$userActivity->user_id = $photo->user_id;
$userActivity->activity_at = $activityDateTime;
$userActivity->type = $type;
$userActivity->photo_id = $photo->id;
$userActivity->save();
}
private function processPhotoAnalyseMessage(QueueItem $queueItem)
{
$this->output->writeln(sprintf('Analysing photo ID %l (batch: %s)', $queueItem->photo_id, $queueItem->batch_reference));
$photo = Photo::where('id', $queueItem->photo_id)->first();
if (is_null($photo))
{
$this->output->writeln('Photo does not exist; skipping');
}
/* IF CHANGING THIS LOGIC, ALSO CHECK PhotoController::analyse */
$photoService = new PhotoService($photo);
$photoService->analyse($queueItem->batch_reference);
// Log an activity record for the user's feed (remove an existing one as the date may have changed)
$this->removeExistingActivityRecords($photo, 'photo.taken');
if (!is_null($photo->taken_at))
{
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
}
}
private function removeExistingActivityRecords(Photo $photo, $type)
{
$existingFeedRecords = UserActivity::where([
'user_id' => $photo->user_id,
'photo_id' => $photo->id,
'type' => $type
])->get();
foreach ($existingFeedRecords as $existingFeedRecord)
{
$existingFeedRecord->delete();
}
}
}

16
app/Helpers/ConfigHelper.php
View File

@ -119,6 +119,12 @@ class ConfigHelper
'photo_comments_allowed_html' => 'p,div,span,a,b,i,u',
'photo_comments_thread_depth' => 3,
'public_statistics' => true,
'rabbitmq_enabled' => false,
'rabbitmq_server' => 'localhost',
'rabbitmq_password' => encrypt('guest'),
'rabbitmq_port' => 5672,
'rabbitmq_queue' => 'blue_twilight',
'rabbitmq_username' => 'guest',
'recaptcha_enabled_registration' => false,
'recaptcha_secret_key' => '',
'recaptcha_site_key' => '',
@ -194,6 +200,16 @@ class ConfigHelper
return $config;
}
public function isImageProcessingQueueEnabled()
{
return $this->get('rabbitmq_enabled') &&
!empty($this->get('rabbitmq_server')) &&
!empty($this->get('rabbitmq_port')) &&
!empty($this->get('rabbitmq_username')) &&
!empty($this->get('rabbitmq_password')) &&
!empty($this->get('rabbitmq_queue'));
}
public function isSocialMediaLoginEnabled()
{
return $this->get('social_facebook_login') ||

22
app/Http/Controllers/Admin/DefaultController.php
View File

@ -39,6 +39,7 @@ class DefaultController extends Controller
View::share('is_admin', true);
$this->passwordSettingKeys = [
'rabbitmq_password',
'smtp_password',
'facebook_app_secret',
'google_app_secret',
@ -241,6 +242,7 @@ class DefaultController extends Controller
'hotlink_protection',
'moderate_anonymous_users',
'moderate_known_users',
'rabbitmq_enabled',
'recaptcha_enabled_registration',
'remove_copyright',
'require_email_verification',
@ -262,6 +264,11 @@ class DefaultController extends Controller
'google_app_secret',
'photo_comments_allowed_html',
'photo_comments_thread_depth',
'rabbitmq_server',
'rabbitmq_port',
'rabbitmq_username',
'rabbitmq_password',
'rabbitmq_queue',
'sender_address',
'sender_name',
'smtp_server',
@ -285,10 +292,17 @@ class DefaultController extends Controller
// Bit of a hack when the browser returns an empty password field - meaning the user didn't change it
// - don't touch it!
if (
$key == 'smtp_password' &&
strlen($config->value) > 0 &&
strlen($request->request->get($key)) == 0 &&
strlen($request->request->get('smtp_username')) > 0
(
$key == 'smtp_password' &&
strlen($config->value) > 0 &&
strlen($request->request->get($key)) == 0 &&
strlen($request->request->get('smtp_username')) > 0
) || (
$key == 'rabbitmq_password' &&
strlen($config->value) > 0 &&
strlen($request->request->get($key)) == 0 &&
strlen($request->request->get('rabbitmq_username')) > 0
)
)
{
continue;

76
app/Http/Controllers/Admin/PhotoController.php
View File

@ -6,13 +6,16 @@ use App\Album;
use App\AlbumSources\IAlbumSource;
use App\Facade\Image;
use App\Facade\Theme;
use App\Facade\UserConfig;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper;
use App\Helpers\MiscHelper;
use App\Http\Requests\UpdatePhotosBulkRequest;
use App\Label;
use App\Photo;
use App\QueueItem;
use App\Services\PhotoService;
use App\Services\RabbitMQService;
use App\Upload;
use App\UploadPhoto;
use App\User;
@ -51,19 +54,54 @@ class PhotoController extends Controller
try
{
$photoService = new PhotoService($photo);
$photoService->analyse($queue_token);
// Log an activity record for the user's feed (remove an existing one as the date may have changed)
$this->removeExistingActivityRecords($photo, 'photo.taken');
if (!is_null($photo->taken_at))
if (UserConfig::isImageProcessingQueueEnabled())
{
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
}
// Find the last record that is analysing this photo
$photoQueueItem = QueueItem::where('photo_id', $photo->id)
->orderBy('queued_at', 'desc')
->limit(1)
->first();
$result['is_successful'] = true;
$timeToWait = 60;
$timeWaited = 0;
$continueToMonitor = true;
while ($continueToMonitor && $timeWaited < $timeToWait)
{
$continueToMonitor = is_null($photoQueueItem->completed_at);
if ($continueToMonitor)
{
sleep(1);
$timeWaited++;
$photoQueueItem = QueueItem::where('id', $photoQueueItem->id)->first();
$continueToMonitor = is_null($photoQueueItem->completed_at);
}
}
$result['is_successful'] = !is_null($photoQueueItem->completed_at);
if (!$result['is_successful'])
{
$result['message'] = 'Timed out waiting for queue processing.';
}
}
else
{
/* IF CHANGING THIS LOGIC, ALSO CHECK ProcessQueueCommand::processPhotoAnalyseMessage */
$photoService = new PhotoService($photo);
$photoService->analyse($queue_token);
// Log an activity record for the user's feed (remove an existing one as the date may have changed)
$this->removeExistingActivityRecords($photo, 'photo.taken');
if (!is_null($photo->taken_at))
{
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
}
$result['is_successful'] = true;
}
}
catch (\Exception $ex)
{
@ -304,6 +342,22 @@ class PhotoController extends Controller
// Log an activity record for the user's feed
$this->createActivityRecord($photo, 'photo.uploaded');
// If queueing is enabled, store the photo in the queue now
if (UserConfig::isImageProcessingQueueEnabled())
{
$queueItem = new QueueItem([
'batch_reference' => $queueUid,
'action_type' => 'photo.analyse',
'album_id' => $photo->album_id,
'photo_id' => $photo->id,
'queued_at' => new \DateTime()
]);
$queueItem->save();
$rabbitmq = new RabbitMQService();
$rabbitmq->queueItem($queueItem);
}
$isSuccessful = true;
}
}

21
app/QueueItem.php Normal file
View File

@ -0,0 +1,21 @@
<?php
namespace App;
use Illuminate\Database\Eloquent\Model;
class QueueItem extends Model
{
/**
* The attributes that are mass assignable.
*
* @var array
*/
protected $fillable = [
'batch_reference',
'action_type',
'album_id',
'photo_id',
'queued_at'
];
}

95
app/Services/RabbitMQService.php Normal file
View File

@ -0,0 +1,95 @@
<?php
namespace App\Services;
use App\Facade\UserConfig;
use App\QueueItem;
use PhpAmqpLib\Channel\AMQPChannel;
use PhpAmqpLib\Connection\AMQPStreamConnection;
use PhpAmqpLib\Message\AMQPMessage;
class RabbitMQService
{
protected $password;
protected $port;
protected $queue;
protected $server;
protected $username;
/**
* @var AMQPChannel
*/
private $channel;
/**
* @var AMQPStreamConnection
*/
private $connection;
public function __construct()
{
$this->server = UserConfig::get('rabbitmq_server');
$this->port = intval(UserConfig::get('rabbitmq_port'));
$this->username = UserConfig::get('rabbitmq_username');
$this->password = decrypt(UserConfig::get('rabbitmq_password'));
$this->queue = UserConfig::get('rabbitmq_queue');
}
public function queueItem(QueueItem $queueItem)
{
$this->connectAndInit();
try
{
$message = new AMQPMessage(
$queueItem->id,
array('delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT)
);
$this->channel->basic_publish($message, '', $this->queue);
}
finally
{
$this->disconnectAndCleanUp();
}
}
public function waitOnQueue($callback)
{
$this->connectAndInit();
try
{
$this->channel->basic_consume($this->queue, '', false, false, false, false, $callback);
while (count($this->channel->callbacks))
{
$this->channel->wait();
}
}
finally
{
$this->disconnectAndCleanUp();
}
}
private function connectAndInit()
{
$this->connection = new AMQPStreamConnection($this->server, $this->port, $this->username, $this->password);
$this->channel = $this->connection->channel();
$this->channel->queue_declare($this->queue, false, true, false, false);
}
private function disconnectAndCleanUp()
{
$this->channel->close();
$this->connection->close();
$this->channel = null;
$this->connection = null;
}
}

3
composer.json
View File

@ -10,7 +10,8 @@
"rackspace/php-opencloud": "^1.16",
"doctrine/dbal": "^2.5",
"aws/aws-sdk-php": "^3.19",
"laravel/socialite": "^3.0"
"laravel/socialite": "^3.0",
"php-amqplib/php-amqplib": "^2.9"
},
"require-dev": {
"filp/whoops": "~2.0",

990
composer.lock generated
View File

File diff suppressed because it is too large Load Diff

45
database/migrations/2019_07_09_203137_create_queue_items_table.php Normal file
View File

@ -0,0 +1,45 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateQueueItemsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('queue_items', function (Blueprint $table) {
$table->increments('id');
$table->string('batch_reference')->nullable(true);
$table->string('action_type', 20);
$table->unsignedInteger('album_id');
$table->unsignedBigInteger('photo_id');
$table->dateTime('queued_at');
$table->dateTime('completed_at')->nullable(true);
$table->timestamps();
$table->foreign('album_id')
->references('id')->on('albums')
->onDelete('cascade');
$table->foreign('photo_id')
->references('id')->on('photos')
->onDelete('cascade');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('queue_items');
}
}

8
resources/lang/en/admin.php
View File

@ -270,6 +270,14 @@ return [
'comments_tab' => 'Comments',
'default_album_permissions' => 'Default Album Permissions',
'default_album_permissions_intro' => 'Configure a set of permissions to apply to top-level albums that do not have their own permissions, and as a base set of permissions for newly-created albums.',
'image_processing_queue' => 'Queue Intensive Operations in RabbitMQ',
'image_processing_queue_beta' => 'Beta',
'image_processing_queue_enabled' => 'Queue intensive operations in RabbitMQ',
'image_processing_queue_enabled_help' => 'If you enable this option, you will also need to provide the connection details to your RabbitMQ server for this to take effect.',
'image_processing_queue_intro' => 'Blue Twilight can off-load the processing of intensive operations (uploads, image analysis and bulk changes) to a secondary server using RabbitMQ.',
'image_processing_queue_intro_2' => 'This requires a significant amount of configuration outside of Blue Twilight, so it is disabled by default. Only enable it if you know what you are doing.',
'image_processing_queue_intro_3' => 'This is a beta-quality feature. We would appreciate feedback through <a href="https://apps.andysh.uk/aheathershaw/blue-twilight/issues">the project\'s issue tracker</a>.',
'image_processing_tab' => 'Image Processing',
'permissions_cache' => 'Permissions Cache',
'permissions_cache_intro' => 'Blue Twilight maintains the permissions each user has to albums in the database. If you feel these aren\'t correct based on what\'s configured, you can rebuild the cache by clicking the button below.',
'rebuild_permissions_cache' => 'Rebuild Permissions Cache',

13
resources/systemd/blue-twilight-mq.service Normal file
View File

@ -0,0 +1,13 @@
[Unit]
Description=Blue Twilight Processing Queue Runner
[Service]
WorkingDirectory=/data/www/blue-twilight.andysh.dev/
ExecStart=/usr/bin/php artisan queue:process
Restart=on-failure
User=www-data
Group=www-data
Environment=USER=www-data HOME=/var/www
[Install]
WantedBy=multi-user.target

77
resources/views/themes/base/admin/settings.blade.php
View File

@ -20,6 +20,7 @@
{{-- Nav tabs --}}
<ul class="nav nav-tabs" role="tablist">
@include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'general', 'tab_icon' => 'info-circle', 'tab_text' => trans('admin.settings_general_tab')])
@include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'image-processing', 'tab_icon' => 'picture-o', 'tab_text' => trans('admin.settings.image_processing_tab')])
@include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'email', 'tab_icon' => 'envelope', 'tab_text' => trans('admin.settings_email_tab')])
@include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'security', 'tab_icon' => 'lock', 'tab_text' => trans('admin.settings_security_tab')])
@include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'analytics', 'tab_icon' => 'line-chart', 'tab_text' => trans('admin.settings.analytics_tab')])
@ -118,6 +119,82 @@
</fieldset>
</div>
{{-- Image Processing --}}
<div role="tabpanel" class="tab-pane" id="image-processing-tab">
<fieldset>
<legend>@lang('admin.settings.image_processing_queue') <span class="badge badge-warning">@lang('admin.settings.image_processing_queue_beta')</span></legend>
<p>@lang('admin.settings.image_processing_queue_intro')</p>
<p>@lang('admin.settings.image_processing_queue_intro_2')</p>
<div class="alert alert-info mb-5">
<p class="mb-0">@lang('admin.settings.image_processing_queue_intro_3')</p>
</div>
</fieldset>
<div class="form-check mb-3">
<input type="checkbox" class="form-check-input" id="rabbitmq-enabled" name="rabbitmq_enabled" @if (UserConfig::get('rabbitmq_enabled'))checked="checked"@endif>
<label class="form-check-label" for="rabbitmq-enabled">
<strong>@lang('admin.settings.image_processing_queue_enabled')</strong><br/>
<span class="text-muted">@lang('admin.settings.image_processing_queue_enabled_help')</span>
</label>
</div>
<div class="form-group ml-4">
<label class="form-control-label" for="rabbitmq-server">Hostname:</label>
<input type="text" class="form-control{{ $errors->has('rabbitmq_server') ? ' is-invalid' : '' }}" id="rabbitmq-server" name="rabbitmq_server" value="{{ old('rabbitmq_server', $config['rabbitmq_server']) }}">
@if ($errors->has('rabbitmq_server'))
<div class="invalid-feedback">
<strong>{{ $errors->first('rabbitmq_server') }}</strong>
</div>
@endif
</div>
<div class="form-group ml-4">
<label class="form-control-label" for="rabbitmq-port">Port:</label>
<input type="text" class="form-control{{ $errors->has('rabbitmq_port') ? ' is-invalid' : '' }}" id="rabbitmq-port" name="rabbitmq_port" value="{{ old('rabbitmq_port', $config['rabbitmq_port']) }}">
@if ($errors->has('rabbitmq_port'))
<div class="invalid-feedback">
<strong>{{ $errors->first('rabbitmq_port') }}</strong>
</div>
@endif
</div>
<div class="form-group ml-4">
<label class="form-control-label" for="rabbitmq-username">Username:</label>
<input type="text" class="form-control{{ $errors->has('rabbitmq_username') ? ' is-invalid' : '' }}" id="rabbitmq-username" name="rabbitmq_username" value="{{ old('rabbitmq_username', $config['rabbitmq_username']) }}">
@if ($errors->has('rabbitmq_username'))
<div class="invalid-feedback">
<strong>{{ $errors->first('rabbitmq_username') }}</strong>
</div>
@endif
</div>
<div class="form-group ml-4">
<label class="form-control-label" for="rabbitmq-password">Password:</label>
<input type="text" class="form-control{{ $errors->has('rabbitmq_password') ? ' is-invalid' : '' }}" id="rabbitmq-password" name="rabbitmq_password" value="{{ old('rabbitmq_password', $config['rabbitmq_password']) }}">
@if ($errors->has('rabbitmq_password'))
<div class="invalid-feedback">
<strong>{{ $errors->first('rabbitmq_password') }}</strong>
</div>
@endif
</div>
<div class="form-group ml-4">
<label class="form-control-label" for="rabbitmq-queue">Queue:</label>
<input type="text" class="form-control{{ $errors->has('rabbitmq_queue') ? ' is-invalid' : '' }}" id="rabbitmq-queue" name="rabbitmq_queue" value="{{ old('rabbitmq_queue', $config['rabbitmq_queue']) }}">
@if ($errors->has('rabbitmq_queue'))
<div class="invalid-feedback">
<strong>{{ $errors->first('rabbitmq_queue') }}</strong>
</div>
@endif
</div>
</div>
{{-- E-mail --}}
<div role="tabpanel" class="tab-pane" id="email-tab">
<div class="form-group">