Merge branch 'feature/121-rabbitmq-queuing' of aheathershaw/blue-twilight into master

app/Console/Commands/ProcessQueueCommand.php

@@ -0,0 +1,158 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Facade\UserConfig;
+use App\Photo;
+use App\QueueItem;
+use App\Services\PhotoService;
+use App\Services\RabbitMQService;
+use App\UserActivity;
+use Illuminate\Console\Command;
+
+class ProcessQueueCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'queue:process';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Processes items in the processing queue.';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        if (!UserConfig::isImageProcessingQueueEnabled())
+        {
+            $this->output->error('The image processing queue is not enabled');
+        }
+
+        $rabbitmq = new RabbitMQService();
+
+        $this->output->writeln('Monitoring queue');
+
+        $rabbitmq->waitOnQueue([$this, 'processQueueItem']);
+    }
+
+    /**
+     * Processes a single item from the queue.
+     *
+     * @param $msg
+     * @return void
+     */
+    public function processQueueItem($msg)
+    {
+        $queueItemID = intval($msg->body);
+
+        $this->output->writeln(sprintf('Processing queue item %d', $queueItemID));
+
+        /** @var QueueItem $queueItem */
+        $queueItem = QueueItem::where('id', $queueItemID)->first();
+        if (is_null($queueItem))
+        {
+            $this->output->writeln('Queue item does not exist; skipping');
+            $msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
+            return;
+        }
+
+        try
+        {
+            switch (strtolower($queueItem->action_type))
+            {
+                case 'photo.analyse':
+                    $this->processPhotoAnalyseMessage($queueItem);
+                    break;
+
+                default:
+                    $this->output->writeln(sprintf('Action %s is not recognised, skipping', $queueItem->action_type));
+                    return;
+            }
+
+            $queueItem->completed_at = new \DateTime();
+            $queueItem->save();
+        }
+        catch (\Exception $ex)
+        {
+            $this->output->error($ex->getMessage());
+        }
+        finally
+        {
+            $msg->delivery_info['channel']->basic_ack($msg->delivery_info['delivery_tag']);
+        }
+    }
+
+    private function createActivityRecord(Photo $photo, $type, $activityDateTime = null)
+    {
+        if (is_null($activityDateTime))
+        {
+            $activityDateTime = new \DateTime();
+        }
+
+        $userActivity = new UserActivity();
+        $userActivity->user_id = $photo->user_id;
+        $userActivity->activity_at = $activityDateTime;
+        $userActivity->type = $type;
+        $userActivity->photo_id = $photo->id;
+        $userActivity->save();
+    }
+
+    private function processPhotoAnalyseMessage(QueueItem $queueItem)
+    {
+        $this->output->writeln(sprintf('Analysing photo ID %l (batch: %s)', $queueItem->photo_id, $queueItem->batch_reference));
+
+        $photo = Photo::where('id', $queueItem->photo_id)->first();
+
+        if (is_null($photo))
+        {
+            $this->output->writeln('Photo does not exist; skipping');
+        }
+
+        /* IF CHANGING THIS LOGIC, ALSO CHECK PhotoController::analyse */
+        $photoService = new PhotoService($photo);
+        $photoService->analyse($queueItem->batch_reference);
+
+        // Log an activity record for the user's feed (remove an existing one as the date may have changed)
+        $this->removeExistingActivityRecords($photo, 'photo.taken');
+
+        if (!is_null($photo->taken_at))
+        {
+            // Log an activity record for the user's feed
+            $this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
+        }
+    }
+
+    private function removeExistingActivityRecords(Photo $photo, $type)
+    {
+        $existingFeedRecords = UserActivity::where([
+            'user_id' => $photo->user_id,
+            'photo_id' => $photo->id,
+            'type' => $type
+        ])->get();
+
+        foreach ($existingFeedRecords as $existingFeedRecord)
+        {
+            $existingFeedRecord->delete();
+        }
+    }
+}

app/Helpers/ConfigHelper.php

@@ -119,6 +119,12 @@ class ConfigHelper
             'photo_comments_allowed_html' => 'p,div,span,a,b,i,u',
             'photo_comments_thread_depth' => 3,
             'public_statistics' => true,
+            'rabbitmq_enabled' => false,
+            'rabbitmq_server' => 'localhost',
+            'rabbitmq_password' => encrypt('guest'),
+            'rabbitmq_port' => 5672,
+            'rabbitmq_queue' => 'blue_twilight',
+            'rabbitmq_username' => 'guest',
             'recaptcha_enabled_registration' => false,
             'recaptcha_secret_key' => '',
             'recaptcha_site_key' => '',
@@ -194,6 +200,16 @@ class ConfigHelper
         return $config;
     }
 
+    public function isImageProcessingQueueEnabled()
+    {
+        return $this->get('rabbitmq_enabled') &&
+            !empty($this->get('rabbitmq_server')) &&
+            !empty($this->get('rabbitmq_port')) &&
+            !empty($this->get('rabbitmq_username')) &&
+            !empty($this->get('rabbitmq_password')) &&
+            !empty($this->get('rabbitmq_queue'));
+    }
+
     public function isSocialMediaLoginEnabled()
     {
         return  $this->get('social_facebook_login') ||

app/Http/Controllers/Admin/DefaultController.php

@@ -39,6 +39,7 @@ class DefaultController extends Controller
         View::share('is_admin', true);
 
         $this->passwordSettingKeys = [
+            'rabbitmq_password',
             'smtp_password',
             'facebook_app_secret',
             'google_app_secret',
@@ -241,6 +242,7 @@ class DefaultController extends Controller
             'hotlink_protection',
             'moderate_anonymous_users',
             'moderate_known_users',
+            'rabbitmq_enabled',
             'recaptcha_enabled_registration',
             'remove_copyright',
             'require_email_verification',
@@ -262,6 +264,11 @@ class DefaultController extends Controller
             'google_app_secret',
             'photo_comments_allowed_html',
             'photo_comments_thread_depth',
+            'rabbitmq_server',
+            'rabbitmq_port',
+            'rabbitmq_username',
+            'rabbitmq_password',
+            'rabbitmq_queue',
             'sender_address',
             'sender_name',
             'smtp_server',
@@ -285,10 +292,17 @@ class DefaultController extends Controller
             // Bit of a hack when the browser returns an empty password field - meaning the user didn't change it
             // - don't touch it!
             if (
-                $key == 'smtp_password' &&
+                (
-                strlen($config->value) > 0 &&
+                    $key == 'smtp_password' &&
-                strlen($request->request->get($key)) == 0 &&
+                    strlen($config->value) > 0 &&
-                strlen($request->request->get('smtp_username')) > 0
+                    strlen($request->request->get($key)) == 0 &&
+                    strlen($request->request->get('smtp_username')) > 0
+                ) || (
+                    $key == 'rabbitmq_password' &&
+                    strlen($config->value) > 0 &&
+                    strlen($request->request->get($key)) == 0 &&
+                    strlen($request->request->get('rabbitmq_username')) > 0
+                )
             )
             {
                 continue;

app/Http/Controllers/Admin/PhotoController.php

@@ -6,13 +6,16 @@ use App\Album;
 use App\AlbumSources\IAlbumSource;
 use App\Facade\Image;
 use App\Facade\Theme;
+use App\Facade\UserConfig;
 use App\Helpers\FileHelper;
 use App\Helpers\ImageHelper;
 use App\Helpers\MiscHelper;
 use App\Http\Requests\UpdatePhotosBulkRequest;
 use App\Label;
 use App\Photo;
+use App\QueueItem;
 use App\Services\PhotoService;
+use App\Services\RabbitMQService;
 use App\Upload;
 use App\UploadPhoto;
 use App\User;
@@ -51,19 +54,54 @@ class PhotoController extends Controller
 
         try
         {
-            $photoService = new PhotoService($photo);
+            if (UserConfig::isImageProcessingQueueEnabled())
-            $photoService->analyse($queue_token);
-
-            // Log an activity record for the user's feed (remove an existing one as the date may have changed)
-            $this->removeExistingActivityRecords($photo, 'photo.taken');
-
-            if (!is_null($photo->taken_at))
             {
-                // Log an activity record for the user's feed
+                // Find the last record that is analysing this photo
-                $this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
+                $photoQueueItem = QueueItem::where('photo_id', $photo->id)
-            }
+                    ->orderBy('queued_at', 'desc')
+                    ->limit(1)
+                    ->first();
 
-            $result['is_successful'] = true;
+                $timeToWait = 60;
+                $timeWaited = 0;
+                $continueToMonitor = true;
+
+                while ($continueToMonitor && $timeWaited < $timeToWait)
+                {
+                    $continueToMonitor = is_null($photoQueueItem->completed_at);
+                    if ($continueToMonitor)
+                    {
+                        sleep(1);
+                        $timeWaited++;
+
+                        $photoQueueItem = QueueItem::where('id', $photoQueueItem->id)->first();
+                        $continueToMonitor = is_null($photoQueueItem->completed_at);
+                    }
+                }
+
+                $result['is_successful'] = !is_null($photoQueueItem->completed_at);
+                if (!$result['is_successful'])
+                {
+                    $result['message'] = 'Timed out waiting for queue processing.';
+                }
+            }
+            else
+            {
+                /* IF CHANGING THIS LOGIC, ALSO CHECK ProcessQueueCommand::processPhotoAnalyseMessage */
+                $photoService = new PhotoService($photo);
+                $photoService->analyse($queue_token);
+
+                // Log an activity record for the user's feed (remove an existing one as the date may have changed)
+                $this->removeExistingActivityRecords($photo, 'photo.taken');
+
+                if (!is_null($photo->taken_at))
+                {
+                    // Log an activity record for the user's feed
+                    $this->createActivityRecord($photo, 'photo.taken', $photo->taken_at);
+                }
+
+                $result['is_successful'] = true;
+            }
         }
         catch (\Exception $ex)
         {
@@ -304,6 +342,22 @@ class PhotoController extends Controller
                 // Log an activity record for the user's feed
                 $this->createActivityRecord($photo, 'photo.uploaded');
 
+                // If queueing is enabled, store the photo in the queue now
+                if (UserConfig::isImageProcessingQueueEnabled())
+                {
+                    $queueItem = new QueueItem([
+                        'batch_reference' => $queueUid,
+                        'action_type' => 'photo.analyse',
+                        'album_id' => $photo->album_id,
+                        'photo_id' => $photo->id,
+                        'queued_at' => new \DateTime()
+                    ]);
+                    $queueItem->save();
+
+                    $rabbitmq = new RabbitMQService();
+                    $rabbitmq->queueItem($queueItem);
+                }
+
                 $isSuccessful = true;
             }
         }

app/QueueItem.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class QueueItem extends Model
+{
+    /**
+     * The attributes that are mass assignable.
+     *
+     * @var array
+     */
+    protected $fillable = [
+        'batch_reference',
+        'action_type',
+        'album_id',
+        'photo_id',
+        'queued_at'
+    ];
+}

app/Services/RabbitMQService.php

@@ -0,0 +1,95 @@
+<?php
+
+namespace App\Services;
+
+use App\Facade\UserConfig;
+use App\QueueItem;
+use PhpAmqpLib\Channel\AMQPChannel;
+use PhpAmqpLib\Connection\AMQPStreamConnection;
+use PhpAmqpLib\Message\AMQPMessage;
+
+class RabbitMQService
+{
+    protected $password;
+
+    protected $port;
+
+    protected $queue;
+
+    protected $server;
+
+    protected $username;
+
+    /**
+     * @var AMQPChannel
+     */
+    private $channel;
+
+    /**
+     * @var AMQPStreamConnection
+     */
+    private $connection;
+
+    public function __construct()
+    {
+        $this->server = UserConfig::get('rabbitmq_server');
+        $this->port = intval(UserConfig::get('rabbitmq_port'));
+        $this->username = UserConfig::get('rabbitmq_username');
+        $this->password = decrypt(UserConfig::get('rabbitmq_password'));
+        $this->queue = UserConfig::get('rabbitmq_queue');
+    }
+
+    public function queueItem(QueueItem $queueItem)
+    {
+        $this->connectAndInit();
+
+        try
+        {
+            $message = new AMQPMessage(
+                $queueItem->id,
+                array('delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT)
+            );
+            $this->channel->basic_publish($message, '', $this->queue);
+        }
+        finally
+        {
+            $this->disconnectAndCleanUp();
+        }
+    }
+
+    public function waitOnQueue($callback)
+    {
+        $this->connectAndInit();
+
+        try
+        {
+            $this->channel->basic_consume($this->queue, '', false, false, false, false, $callback);
+
+            while (count($this->channel->callbacks))
+            {
+                $this->channel->wait();
+            }
+        }
+        finally
+        {
+            $this->disconnectAndCleanUp();
+        }
+    }
+
+    private function connectAndInit()
+    {
+        $this->connection = new AMQPStreamConnection($this->server, $this->port, $this->username, $this->password);
+        $this->channel = $this->connection->channel();
+
+        $this->channel->queue_declare($this->queue, false, true, false, false);
+    }
+
+    private function disconnectAndCleanUp()
+    {
+        $this->channel->close();
+        $this->connection->close();
+
+        $this->channel = null;
+        $this->connection = null;
+    }
+}

composer.json

@@ -10,7 +10,8 @@
         "rackspace/php-opencloud": "^1.16",
         "doctrine/dbal": "^2.5",
         "aws/aws-sdk-php": "^3.19",
-        "laravel/socialite": "^3.0"
+        "laravel/socialite": "^3.0",
+        "php-amqplib/php-amqplib": "^2.9"
     },
     "require-dev": {
         "filp/whoops": "~2.0",

database/migrations/2019_07_09_203137_create_queue_items_table.php

@@ -0,0 +1,45 @@
+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateQueueItemsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('queue_items', function (Blueprint $table) {
+            $table->increments('id');
+            $table->string('batch_reference')->nullable(true);
+            $table->string('action_type', 20);
+            $table->unsignedInteger('album_id');
+            $table->unsignedBigInteger('photo_id');
+            $table->dateTime('queued_at');
+            $table->dateTime('completed_at')->nullable(true);
+            $table->timestamps();
+
+            $table->foreign('album_id')
+                ->references('id')->on('albums')
+                ->onDelete('cascade');
+
+            $table->foreign('photo_id')
+                ->references('id')->on('photos')
+                ->onDelete('cascade');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('queue_items');
+    }
+}

resources/lang/en/admin.php

@@ -270,6 +270,14 @@ return [
         'comments_tab' => 'Comments',
         'default_album_permissions' => 'Default Album Permissions',
         'default_album_permissions_intro' => 'Configure a set of permissions to apply to top-level albums that do not have their own permissions, and as a base set of permissions for newly-created albums.',
+        'image_processing_queue' => 'Queue Intensive Operations in RabbitMQ',
+        'image_processing_queue_beta' => 'Beta',
+        'image_processing_queue_enabled' => 'Queue intensive operations in RabbitMQ',
+        'image_processing_queue_enabled_help' => 'If you enable this option, you will also need to provide the connection details to your RabbitMQ server for this to take effect.',
+        'image_processing_queue_intro' => 'Blue Twilight can off-load the processing of intensive operations (uploads, image analysis and bulk changes) to a secondary server using RabbitMQ.',
+        'image_processing_queue_intro_2' => 'This requires a significant amount of configuration outside of Blue Twilight, so it is disabled by default. Only enable it if you know what you are doing.',
+        'image_processing_queue_intro_3' => 'This is a beta-quality feature. We would appreciate feedback through <a href="https://apps.andysh.uk/aheathershaw/blue-twilight/issues">the project\'s issue tracker</a>.',
+        'image_processing_tab' => 'Image Processing',
         'permissions_cache' => 'Permissions Cache',
         'permissions_cache_intro' => 'Blue Twilight maintains the permissions each user has to albums in the database. If you feel these aren\'t correct based on what\'s configured, you can rebuild the cache by clicking the button below.',
         'rebuild_permissions_cache' => 'Rebuild Permissions Cache',

resources/systemd/blue-twilight-mq.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Blue Twilight Processing Queue Runner
+
+[Service]
+WorkingDirectory=/data/www/blue-twilight.andysh.dev/
+ExecStart=/usr/bin/php artisan queue:process
+Restart=on-failure
+User=www-data
+Group=www-data
+Environment=USER=www-data HOME=/var/www
+
+[Install]
+WantedBy=multi-user.target

resources/views/themes/base/admin/settings.blade.php

@@ -20,6 +20,7 @@
                         {{-- Nav tabs --}}
                         <ul class="nav nav-tabs" role="tablist">
                             @include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'general', 'tab_icon' => 'info-circle', 'tab_text' => trans('admin.settings_general_tab')])
+                            @include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'image-processing', 'tab_icon' => 'picture-o', 'tab_text' => trans('admin.settings.image_processing_tab')])
                             @include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'email', 'tab_icon' => 'envelope', 'tab_text' => trans('admin.settings_email_tab')])
                             @include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'security', 'tab_icon' => 'lock', 'tab_text' => trans('admin.settings_security_tab')])
                             @include(Theme::viewName('partials.tab'), ['active_tab' => 'general', 'tab_name' => 'analytics', 'tab_icon' => 'line-chart', 'tab_text' => trans('admin.settings.analytics_tab')])
@@ -118,6 +119,82 @@
                                 </fieldset>
                             </div>
 
+                            {{-- Image Processing --}}
+                            <div role="tabpanel" class="tab-pane" id="image-processing-tab">
+                                <fieldset>
+                                    <legend>@lang('admin.settings.image_processing_queue') <span class="badge badge-warning">@lang('admin.settings.image_processing_queue_beta')</span></legend>
+                                    <p>@lang('admin.settings.image_processing_queue_intro')</p>
+                                    <p>@lang('admin.settings.image_processing_queue_intro_2')</p>
+
+                                    <div class="alert alert-info mb-5">
+                                        <p class="mb-0">@lang('admin.settings.image_processing_queue_intro_3')</p>
+                                    </div>
+                                </fieldset>
+
+                                <div class="form-check mb-3">
+                                    <input type="checkbox" class="form-check-input" id="rabbitmq-enabled" name="rabbitmq_enabled" @if (UserConfig::get('rabbitmq_enabled'))checked="checked"@endif>
+                                    <label class="form-check-label" for="rabbitmq-enabled">
+                                        <strong>@lang('admin.settings.image_processing_queue_enabled')</strong><br/>
+                                        <span class="text-muted">@lang('admin.settings.image_processing_queue_enabled_help')</span>
+                                    </label>
+                                </div>
+
+                                <div class="form-group ml-4">
+                                    <label class="form-control-label" for="rabbitmq-server">Hostname:</label>
+                                    <input type="text" class="form-control{{ $errors->has('rabbitmq_server') ? ' is-invalid' : '' }}" id="rabbitmq-server" name="rabbitmq_server" value="{{ old('rabbitmq_server', $config['rabbitmq_server']) }}">
+
+                                    @if ($errors->has('rabbitmq_server'))
+                                        <div class="invalid-feedback">
+                                            <strong>{{ $errors->first('rabbitmq_server') }}</strong>
+                                        </div>
+                                    @endif
+                                </div>
+
+                                <div class="form-group ml-4">
+                                    <label class="form-control-label" for="rabbitmq-port">Port:</label>
+                                    <input type="text" class="form-control{{ $errors->has('rabbitmq_port') ? ' is-invalid' : '' }}" id="rabbitmq-port" name="rabbitmq_port" value="{{ old('rabbitmq_port', $config['rabbitmq_port']) }}">
+
+                                    @if ($errors->has('rabbitmq_port'))
+                                        <div class="invalid-feedback">
+                                            <strong>{{ $errors->first('rabbitmq_port') }}</strong>
+                                        </div>
+                                    @endif
+                                </div>
+
+                                <div class="form-group ml-4">
+                                    <label class="form-control-label" for="rabbitmq-username">Username:</label>
+                                    <input type="text" class="form-control{{ $errors->has('rabbitmq_username') ? ' is-invalid' : '' }}" id="rabbitmq-username" name="rabbitmq_username" value="{{ old('rabbitmq_username', $config['rabbitmq_username']) }}">
+
+                                    @if ($errors->has('rabbitmq_username'))
+                                        <div class="invalid-feedback">
+                                            <strong>{{ $errors->first('rabbitmq_username') }}</strong>
+                                        </div>
+                                    @endif
+                                </div>
+
+                                <div class="form-group ml-4">
+                                    <label class="form-control-label" for="rabbitmq-password">Password:</label>
+                                    <input type="text" class="form-control{{ $errors->has('rabbitmq_password') ? ' is-invalid' : '' }}" id="rabbitmq-password" name="rabbitmq_password" value="{{ old('rabbitmq_password', $config['rabbitmq_password']) }}">
+
+                                    @if ($errors->has('rabbitmq_password'))
+                                        <div class="invalid-feedback">
+                                            <strong>{{ $errors->first('rabbitmq_password') }}</strong>
+                                        </div>
+                                    @endif
+                                </div>
+
+                                <div class="form-group ml-4">
+                                    <label class="form-control-label" for="rabbitmq-queue">Queue:</label>
+                                    <input type="text" class="form-control{{ $errors->has('rabbitmq_queue') ? ' is-invalid' : '' }}" id="rabbitmq-queue" name="rabbitmq_queue" value="{{ old('rabbitmq_queue', $config['rabbitmq_queue']) }}">
+
+                                    @if ($errors->has('rabbitmq_queue'))
+                                        <div class="invalid-feedback">
+                                            <strong>{{ $errors->first('rabbitmq_queue') }}</strong>
+                                        </div>
+                                    @endif
+                                </div>
+                            </div>
+
                             {{-- E-mail --}}
                             <div role="tabpanel" class="tab-pane" id="email-tab">
                                 <div class="form-group">