BLUE-8: Modified the upload/analysis functionality to work strictly with local files, so only the final results are uploaded to OpenStack, saving on bandwidth

This commit is contained in:
Andy Heathershaw 2016-10-28 05:30:57 +01:00
parent 9141398da8
commit aadc39684f
8 changed files with 97 additions and 78 deletions

View File

@ -0,0 +1,48 @@
<?php
namespace App\Helpers;
use Illuminate\Http\File;
use Illuminate\Http\UploadedFile;
class FileHelper
{
public static function getQueuePath($queueUid)
{
$path = join(DIRECTORY_SEPARATOR, [
dirname(dirname(__DIR__)),
'storage',
'app',
'analysis-queue',
str_replace(['.', '/', '\\'], '', $queueUid)
]);
if (!file_exists($path))
{
mkdir($path, 0755, true);
}
return $path;
}
public static function saveUploadedFile(UploadedFile $uploadedFile, $destinationPath, $overrideFilename = null)
{
$tempFilename = join(DIRECTORY_SEPARATOR, [
$destinationPath,
is_null($overrideFilename) ? MiscHelper::randomString(20) : basename($overrideFilename)
]);
// Only add an extension if an override filename was not given, assume this is present
if (is_null($overrideFilename))
{
$extension = $uploadedFile->guessExtension();
if (!is_null($extension))
{
$tempFilename .= '.' . $extension;
}
}
$uploadedFile->move(dirname($tempFilename), basename($tempFilename));
return new File($tempFilename);
}
}

View File

@ -5,6 +5,7 @@ namespace app\Http\Controllers\Admin;
use App\Album; use App\Album;
use App\Facade\Theme; use App\Facade\Theme;
use App\Facade\UserConfig; use App\Facade\UserConfig;
use App\Helpers\FileHelper;
use App\Helpers\MiscHelper; use App\Helpers\MiscHelper;
use App\Http\Controllers\Controller; use App\Http\Controllers\Controller;
use App\Http\Requests; use App\Http\Requests;
@ -26,7 +27,7 @@ class AlbumController extends Controller
View::share('is_admin', true); View::share('is_admin', true);
} }
public function analyse($id) public function analyse($id, $queue_token)
{ {
$this->authorize('admin-access'); $this->authorize('admin-access');
@ -36,7 +37,7 @@ class AlbumController extends Controller
->orderBy('created_at') ->orderBy('created_at')
->get(); ->get();
return Theme::render('admin.analyse_album', ['album' => $album, 'photos' => $photos]); return Theme::render('admin.analyse_album', ['album' => $album, 'photos' => $photos, 'queue_token' => $queue_token]);
} }
/** /**
@ -184,6 +185,7 @@ class AlbumController extends Controller
'max_post_limit' => $postLimit, 'max_post_limit' => $postLimit,
'max_post_limit_bulk' => $fileUploadOrPostLowerLimit, 'max_post_limit_bulk' => $fileUploadOrPostLowerLimit,
'photos' => $photos, 'photos' => $photos,
'queue_token' => MiscHelper::randomString(),
'success' => $request->session()->get('success'), 'success' => $request->session()->get('success'),
'warning' => $request->session()->get('warning') 'warning' => $request->session()->get('warning')
]); ]);

View File

@ -6,6 +6,7 @@ use App\Album;
use App\AlbumSources\IAlbumSource; use App\AlbumSources\IAlbumSource;
use App\Facade\Image; use App\Facade\Image;
use App\Facade\Theme; use App\Facade\Theme;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper; use App\Helpers\ImageHelper;
use App\Helpers\MiscHelper; use App\Helpers\MiscHelper;
use App\Photo; use App\Photo;
@ -30,7 +31,7 @@ class PhotoController extends Controller
View::share('is_admin', true); View::share('is_admin', true);
} }
public function analyse($photoId) public function analyse($photoId, $queue_token)
{ {
$this->authorize('admin-access'); $this->authorize('admin-access');
@ -47,7 +48,7 @@ class PhotoController extends Controller
try try
{ {
$photoService = new PhotoService($photo); $photoService = new PhotoService($photo);
$photoService->analyse(); $photoService->analyse($queue_token);
$result['is_successful'] = true; $result['is_successful'] = true;
} }
@ -222,6 +223,15 @@ class PhotoController extends Controller
$album = $this->loadAlbum($request->get('album_id')); $album = $this->loadAlbum($request->get('album_id'));
$isSuccessful = false; $isSuccessful = false;
// Create the folder to hold the analysis results if not already present
$queueUid = $request->get('queue_token');
if (strlen($queueUid) == 0)
{
throw new \Exception('No queue_token value was provided!');
}
$queueFolder = FileHelper::getQueuePath($queueUid);
foreach ($photoFiles as $photoFile) foreach ($photoFiles as $photoFile)
{ {
$photoFile = UploadedFile::createFromBase($photoFile); $photoFile = UploadedFile::createFromBase($photoFile);
@ -232,7 +242,7 @@ class PhotoController extends Controller
else else
{ {
/** @var File $savedFile */ /** @var File $savedFile */
$savedFile = $album->getAlbumSource()->saveUploadedPhoto($photoFile); $savedFile = FileHelper::saveUploadedFile($photoFile, $queueFolder);
$photo = new Photo(); $photo = new Photo();
$photo->album_id = $album->id; $photo->album_id = $album->id;
@ -256,7 +266,8 @@ class PhotoController extends Controller
else else
{ {
return redirect(route('albums.analyse', [ return redirect(route('albums.analyse', [
'id' => $album->id 'id' => $album->id,
'queue_token' => $queueUid
])); ]));
} }
} }
@ -276,9 +287,14 @@ class PhotoController extends Controller
return redirect(route('albums.show', ['id' => $album->id])); return redirect(route('albums.show', ['id' => $album->id]));
} }
// Create a temporary folder to hold the extracted files // Create the folder to hold the analysis results if not already present
$tempFolder = sprintf('%s/btw_upload_%s', env('TEMP_FOLDER', '/tmp'), MiscHelper::randomString()); $queueUid = $request->get('queue_token');
mkdir($tempFolder); if (strlen($queueUid) == 0)
{
throw new \Exception('No queue_token value was provided!');
}
$queueFolder = FileHelper::getQueuePath($queueUid);
$mimeType = strtolower($archiveFile->getMimeType()); $mimeType = strtolower($archiveFile->getMimeType());
switch ($mimeType) switch ($mimeType)
@ -286,7 +302,7 @@ class PhotoController extends Controller
case 'application/zip': case 'application/zip':
$zip = new \ZipArchive(); $zip = new \ZipArchive();
$zip->open($archiveFile->getPathname()); $zip->open($archiveFile->getPathname());
$zip->extractTo($tempFolder); $zip->extractTo($queueFolder);
$zip->close(); $zip->close();
break; break;
@ -295,7 +311,7 @@ class PhotoController extends Controller
return redirect(route('albums.show', ['id' => $album->id])); return redirect(route('albums.show', ['id' => $album->id]));
} }
$di = new \RecursiveDirectoryIterator($tempFolder, \RecursiveDirectoryIterator::SKIP_DOTS); $di = new \RecursiveDirectoryIterator($queueFolder, \RecursiveDirectoryIterator::SKIP_DOTS);
$recursive = new \RecursiveIteratorIterator($di); $recursive = new \RecursiveIteratorIterator($di);
/** @var \SplFileInfo $fileInfo */ /** @var \SplFileInfo $fileInfo */
@ -336,10 +352,11 @@ class PhotoController extends Controller
$photo->save(); $photo->save();
} }
@rmdir($tempFolder); @rmdir($queueFolder);
return redirect(route('albums.analyse', [ return redirect(route('albums.analyse', [
'id' => $album->id 'id' => $album->id,
'queue_token' => $queueUid
])); ]));
} }

View File

@ -4,6 +4,7 @@ namespace App\Services;
use App\Album; use App\Album;
use App\AlbumSources\IAlbumSource; use App\AlbumSources\IAlbumSource;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper; use App\Helpers\ImageHelper;
use App\Helpers\ThemeHelper; use App\Helpers\ThemeHelper;
use App\Photo; use App\Photo;
@ -48,13 +49,15 @@ class PhotoService
$this->themeHelper = new ThemeHelper(); $this->themeHelper = new ThemeHelper();
} }
public function analyse() public function analyse($queueToken)
{ {
/** @var Album $album */ /** @var Album $album */
$album = $this->photo->album; $album = $this->photo->album;
$albumSource = $album->getAlbumSource();
$photoFile = $albumSource->getPathToPhoto($this->photo); $photoFile = join(DIRECTORY_SEPARATOR, [
FileHelper::getQueuePath($queueToken),
$this->photo->storage_file_name
]);
$imageInfo = null; $imageInfo = null;
$originalPhotoResource = $this->imageHelper->openImage($photoFile, $imageInfo); $originalPhotoResource = $this->imageHelper->openImage($photoFile, $imageInfo);

View File

@ -434,11 +434,12 @@ function StorageLocationsViewModel() {
/** /**
* This model is used by admin/show_album.blade.php to handle photo uploads. * This model is used by admin/show_album.blade.php to handle photo uploads.
* @param album_id ID of the album the photos are being uploaded to * @param album_id ID of the album the photos are being uploaded to
* @param queue_token Unique token of the upload queue to save the photos to
* @param language Array containing language strings * @param language Array containing language strings
* @param urls Array containing URLs * @param urls Array containing URLs
* @constructor * @constructor
*/ */
function UploadPhotosViewModel(album_id, language, urls) { function UploadPhotosViewModel(album_id, queue_token, language, urls) {
var self = this; var self = this;
self.currentStatus = ko.observable(''); self.currentStatus = ko.observable('');
@ -510,6 +511,7 @@ function UploadPhotosViewModel(album_id, language, urls) {
self.uploadFile = function uploadImageFile(formObject, imageFile) { self.uploadFile = function uploadImageFile(formObject, imageFile) {
var formData = new FormData(); var formData = new FormData();
formData.append('album_id', album_id); formData.append('album_id', album_id);
formData.append('queue_token', queue_token);
formData.append('photo[]', imageFile, imageFile.name); formData.append('photo[]', imageFile, imageFile.name);
$.ajax( $.ajax(

View File

@ -56,65 +56,10 @@
viewModel.imagesToAnalyse.push(new AnalyseImageViewModel({ viewModel.imagesToAnalyse.push(new AnalyseImageViewModel({
'id': '{{ $photo->id }}', 'id': '{{ $photo->id }}',
'name': '{!! addslashes($photo->name) !!}', 'name': '{!! addslashes($photo->name) !!}',
'url': '{{ route('photos.analyse', ['id' => $photo->id]) }}' 'url': '{{ route('photos.analyse', ['id' => $photo->id, 'queue_token' => $queue_token]) }}'
})); }));
@endforeach @endforeach
ko.applyBindings(viewModel); ko.applyBindings(viewModel);
/*
$(document).ready(function() {
number_total = $('#file-list p').length;
if (number_total == 0) {
$('#status-panel').hide();
$('#complete-panel').show();
}
else
{
$('#file-list p').each(function (index, element) {
var photo_id = $(element).data('photo-id');
var url = '{{ route('photos.analyse', ['id' => 0]) }}';
url = url.replace(/0$/, photo_id);
$.ajax(
url,
{
complete: function () {
redrawProgressBar();
if (number_successful + number_error >= number_total) {
$('#status-panel').hide();
$('#complete-panel').show();
}
},
dataType: 'json',
error: function (xhr, textStatus, errorThrown) {
$('i', '#file-list p[data-photo-id=' + photo_id + ']')
.addClass('text-danger')
.addClass('fa-times');
number_error++;
},
method: 'POST',
success: function (data) {
if (data.is_successful) {
$('i', '#file-list p[data-photo-id=' + photo_id + ']')
.addClass('text-success')
.addClass('fa-check');
number_successful++;
}
else {
$('i', '#file-list p[data-photo-id=' + photo_id + ']')
.addClass('text-danger')
.addClass('fa-times');
number_error++;
}
}
}
);
});
}
});*/
</script> </script>
@endpush @endpush

View File

@ -89,6 +89,7 @@
<div class="col-sm-5" style="margin-bottom: 20px;"> <div class="col-sm-5" style="margin-bottom: 20px;">
{!! Form::open(['route' => 'photos.store', 'method' => 'POST', 'files' => true, 'id' => 'single-upload-form']) !!} {!! Form::open(['route' => 'photos.store', 'method' => 'POST', 'files' => true, 'id' => 'single-upload-form']) !!}
{!! Form::hidden('album_id', $album->id) !!} {!! Form::hidden('album_id', $album->id) !!}
{!! Form::hidden('queue_token', $queue_token) !!}
<div class="form-group"> <div class="form-group">
{!! Form::file('photo[]', ['class' => 'control-label', 'multiple' => 'multiple', 'id' => 'single-upload-files']) !!} {!! Form::file('photo[]', ['class' => 'control-label', 'multiple' => 'multiple', 'id' => 'single-upload-files']) !!}
@ -120,7 +121,7 @@
</p> </p>
<p data-bind="visible: imagesUploaded() > 0"> <p data-bind="visible: imagesUploaded() > 0">
@lang('admin.upload_file_failed_continue')<br /><br/> @lang('admin.upload_file_failed_continue')<br /><br/>
<a href="{{ route('albums.analyse', ['id' => $album->id]) }}" class="btn btn-primary">@lang('forms.continue_action')</a> <a href="{{ route('albums.analyse', ['id' => $album->id, 'queue_token' => $queue_token]) }}" class="btn btn-primary">@lang('forms.continue_action')</a>
</p> </p>
<ul data-bind="foreach: statusMessages"> <ul data-bind="foreach: statusMessages">
@ -141,6 +142,7 @@
{!! Form::open(['route' => 'photos.storeBulk', 'method' => 'POST', 'files' => true, 'id' => 'bulk-upload-form']) !!} {!! Form::open(['route' => 'photos.storeBulk', 'method' => 'POST', 'files' => true, 'id' => 'bulk-upload-form']) !!}
{!! Form::hidden('album_id', $album->id) !!} {!! Form::hidden('album_id', $album->id) !!}
{!! Form::hidden('queue_token', $queue_token) !!}
<div class="form-group"> <div class="form-group">
{!! Form::file('archive', ['class' => 'control-label']) !!} {!! Form::file('archive', ['class' => 'control-label']) !!}
@ -244,14 +246,14 @@
language.upload_status = '{!! addslashes(trans('admin.upload_file_status_progress')) !!}'; language.upload_status = '{!! addslashes(trans('admin.upload_file_status_progress')) !!}';
var urls = []; var urls = [];
urls.analyse = '{{ route('albums.analyse', ['id' => $album->id]) }}'; urls.analyse = '{{ route('albums.analyse', ['id' => $album->id, 'queue_token' => $queue_token]) }}';
urls.delete_photo = '{{ route('photos.destroy', ['id' => 0]) }}'; urls.delete_photo = '{{ route('photos.destroy', ['id' => 0]) }}';
urls.flip_photo = '{{ route('photos.flip', ['id' => 0, 'horizontal' => -1, 'vertical' => -2]) }}'; urls.flip_photo = '{{ route('photos.flip', ['id' => 0, 'horizontal' => -1, 'vertical' => -2]) }}';
urls.move_photo = '{{ route('photos.move', ['photoId' => 0]) }}'; urls.move_photo = '{{ route('photos.move', ['photoId' => 0]) }}';
urls.regenerate_thumbnails = '{{ route('photos.regenerateThumbnails', ['photoId' => 0]) }}'; urls.regenerate_thumbnails = '{{ route('photos.regenerateThumbnails', ['photoId' => 0]) }}';
urls.rotate_photo = '{{ route('photos.rotate', ['id' => 0, 'angle' => 1]) }}'; urls.rotate_photo = '{{ route('photos.rotate', ['id' => 0, 'angle' => 1]) }}';
var viewModel = new UploadPhotosViewModel('{{ $album->id }}', language, urls); var viewModel = new UploadPhotosViewModel('{{ $album->id }}', '{{ $queue_token }}', language, urls);
var editViewModel = new EditPhotosViewModel('{{ $album->id }}', language, urls); var editViewModel = new EditPhotosViewModel('{{ $album->id }}', language, urls);
// Populate the list of albums in the view model // Populate the list of albums in the view model

View File

@ -21,12 +21,12 @@ Route::group(['prefix' => 'admin'], function () {
Route::get('settings', 'Admin\DefaultController@settings')->name('admin.settings'); Route::get('settings', 'Admin\DefaultController@settings')->name('admin.settings');
// Album management // Album management
Route::get('albums/{id}/analyse', 'Admin\AlbumController@analyse')->name('albums.analyse'); Route::get('albums/{id}/analyse/{queue_token}', 'Admin\AlbumController@analyse')->name('albums.analyse');
Route::get('albums/{id}/delete', 'Admin\AlbumController@delete')->name('albums.delete'); Route::get('albums/{id}/delete', 'Admin\AlbumController@delete')->name('albums.delete');
Route::resource('albums', 'Admin\AlbumController'); Route::resource('albums', 'Admin\AlbumController');
// Photo management // Photo management
Route::post('photos/analyse/{id}', 'Admin\PhotoController@analyse')->name('photos.analyse'); Route::post('photos/analyse/{id}/{queue_token}', 'Admin\PhotoController@analyse')->name('photos.analyse');
Route::post('photos/flip/{photoId}/{horizontal}/{vertical}', 'Admin\PhotoController@flip')->name('photos.flip'); Route::post('photos/flip/{photoId}/{horizontal}/{vertical}', 'Admin\PhotoController@flip')->name('photos.flip');
Route::post('photos/move/{photoId}', 'Admin\PhotoController@move')->name('photos.move'); Route::post('photos/move/{photoId}', 'Admin\PhotoController@move')->name('photos.move');
Route::post('photos/regenerate-thumbnails/{photoId}', 'Admin\PhotoController@regenerateThumbnails')->name('photos.regenerateThumbnails'); Route::post('photos/regenerate-thumbnails/{photoId}', 'Admin\PhotoController@regenerateThumbnails')->name('photos.regenerateThumbnails');