Files are now removed from Dropbox when a photo/album is deleted. Added handling for Dropbox's 429 (retry) error. Added a new admin permission for restricing access to the new services area. Corrected a logic issue with failing images during the analysis process. #106

This commit is contained in:
Andy Heathershaw 2020-04-22 06:56:15 +01:00
parent f17a84f746
commit f80b80540f
11 changed files with 151 additions and 52 deletions

View File

@ -20,7 +20,16 @@ class DropboxSource extends AlbumSourceBase implements IAlbumSource
*/ */
public function deleteAlbumContents() public function deleteAlbumContents()
{ {
// TODO: Implement deleteAlbumContents() method. try
{
$albumPathOnStorage = sprintf('/%s', $this->album->url_alias);
$this->getClient()->deleteFile($albumPathOnStorage);
}
catch (\Exception $ex)
{
// Don't worry too much if the delete fails - the file may have been removed on Dropbox itself
}
} }
/** /**
@ -31,7 +40,16 @@ class DropboxSource extends AlbumSourceBase implements IAlbumSource
*/ */
public function deleteThumbnail(Photo $photo, $thumbnail = null) public function deleteThumbnail(Photo $photo, $thumbnail = null)
{ {
// TODO: Implement deleteThumbnail() method. try
{
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
$this->getClient()->deleteFile($pathOnStorage);
}
catch (\Exception $ex)
{
// Don't worry too much if the delete fails - the file may have been removed on Dropbox itself
}
} }
/** /**

View File

@ -0,0 +1,23 @@
<?php
namespace App\Exceptions;
class DropboxRetryException extends \Exception
{
private $innerException;
/**
* @return mixed
*/
public function getInnerException()
{
return $this->innerException;
}
public function __construct($httpCode, \Exception $innerException)
{
parent::__construct('Dropbox requested to retry the request');
$this->innerException = $innerException;
}
}

View File

@ -35,7 +35,7 @@ class StorageController extends Controller
$storage = Storage::where('id', intval($id))->first(); $storage = Storage::where('id', intval($id))->first();
if (is_null($storage)) if (is_null($storage))
{ {
App::abort(404); return redirect(route('storages.index'));
} }
$externalServiceType = $this->getExternalServiceType($storage); $externalServiceType = $this->getExternalServiceType($storage);
@ -47,26 +47,17 @@ class StorageController extends Controller
} }
$serviceTypeName = trans(sprintf('services.%s', $externalServiceType)); $serviceTypeName = trans(sprintf('services.%s', $externalServiceType));
$viewData = [
'service' => $storage->externalService,
'serviceName' => $serviceTypeName,
'storage' => $storage
];
switch ($externalServiceType) switch ($externalServiceType)
{ {
case ExternalService::DROPBOX: case ExternalService::DROPBOX:
$dropbox = new DropboxService(); $dropbox = new DropboxService();
$viewData['authoriseUrl'] = $dropbox->authoriseUrl($storage); return redirect($dropbox->authoriseUrl($storage));
$viewData['callbackUrl'] = $dropbox->callbackUrl();
break;
default: default:
$request->session()->flash('error', trans('admin.storage_external_service_no_authorisation', ['service_name' => $serviceTypeName])); $request->session()->flash('error', trans('admin.storage_external_service_no_authorisation', ['service_name' => $serviceTypeName]));
return redirect(route('storages.index')); return redirect(route('storages.index'));
} }
return Theme::render('admin.authorise_external_service', $viewData);
} }
/** /**

View File

@ -2,6 +2,7 @@
namespace App\Services; namespace App\Services;
use App\Exceptions\DropboxRetryException;
use App\Storage; use App\Storage;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
@ -43,6 +44,25 @@ class DropboxService
return route('services.authoriseDropbox'); return route('services.authoriseDropbox');
} }
public function deleteFile($pathOnStorage)
{
$dropboxData = ['path' => $pathOnStorage];
$deleteResult = $this->sendRequest(
$this->config['delete_url'],
'POST',
$dropboxData,
[
'http_headers' => [
'Content-Type: application/json'
],
'post_body_is_json' => true
]
);
Log::debug('DropboxService - response to deleteFile.', ['response' => $deleteResult, 'path' => $pathOnStorage]);
}
public function downloadFile($pathOnStorage) public function downloadFile($pathOnStorage)
{ {
$dropboxArgs = ['path' => $pathOnStorage]; $dropboxArgs = ['path' => $pathOnStorage];
@ -80,14 +100,19 @@ class DropboxService
$this->accessToken = $accessToken; $this->accessToken = $accessToken;
} }
public function uploadFile($pathToFileToUpload, $pathToStorage) public function uploadFile($pathToFileToUpload, $pathOnStorage)
{ {
$dropboxArgs = [ $dropboxArgs = [
'path' => $pathToStorage, 'path' => $pathOnStorage,
'mode' => 'add', 'mode' => 'overwrite',
'mute' => true 'mute' => true
]; ];
$shouldRetry = true;
while ($shouldRetry)
{
try
{
$uploadResult = $this->sendRequest( $uploadResult = $this->sendRequest(
$this->config['upload_url'], $this->config['upload_url'],
'POST', 'POST',
@ -100,6 +125,22 @@ class DropboxService
'post_body_is_json' => false 'post_body_is_json' => false
] ]
); );
$shouldRetry = false;
Log::debug('DropboxService - response to uploadFile.', ['response' => $uploadResult, 'path' => $pathOnStorage]);
}
catch (DropboxRetryException $dre)
{
// Retry - leave shouldRetry as true
Log::debug('DropboxService - Dropbox reported a lock/rate limit and requested to retry');
sleep(2);
}
catch (\Exception $ex)
{
$shouldRetry = false;
Log::debug('DropboxService - exception in uploadFile.', ['exception' => $ex->getMessage()]);
}
}
} }
private function convertAuthorisationCodeToToken($authorisationCode, Storage $storage) private function convertAuthorisationCodeToToken($authorisationCode, Storage $storage)
@ -178,29 +219,25 @@ class DropboxService
$ch = $this->getBasicHttpClient($url, $method, $httpHeaders); $ch = $this->getBasicHttpClient($url, $method, $httpHeaders);
Log::info(sprintf('DropboxService - %s: %s', strtoupper($method), $url));
Log::debug('DropboxService - HTTP headers:', $httpHeaders);
if (!is_null($postData)) if (!is_null($postData))
{ {
if ($postOptions['post_body_is_json']) if ($postOptions['post_body_is_json'])
{ {
// Only log a post body if we have one and it's in JSON format (i.e. not a file upload)
Log::debug('DropboxService - Body: ', $postData);
$postData = json_encode($postData); $postData = json_encode($postData);
} }
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData); curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
} }
Log::info(sprintf('%s: %s', strtoupper($method), $url));
Log::debug('HTTP headers:', $httpHeaders);
// Only log a post body if we have one and it's in JSON format (i.e. not a file upload)
if (!is_null($postData) && $postOptions['post_body_is_json'])
{
Log::debug($postData);
}
$result = curl_exec($ch); $result = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
Log::info(sprintf('Received HTTP code %d', $httpCode)); Log::info(sprintf('DropboxService - Received HTTP code %d', $httpCode));
// Only log a result if we have one and it's in JSON format (i.e. not a file download) // Only log a result if we have one and it's in JSON format (i.e. not a file download)
if (!is_null($result) && $result !== false && $postOptions['response_body_is_json']) if (!is_null($result) && $result !== false && $postOptions['response_body_is_json'])
@ -208,15 +245,25 @@ class DropboxService
Log::debug($result); Log::debug($result);
} }
try
{
if ($httpCode != 200 && $httpCode != 304) if ($httpCode != 200 && $httpCode != 304)
{ {
throw new \Exception(sprintf('Exception from Dropbox: %s', $result)); if ($httpCode == 429)
{
throw new DropboxRetryException($httpCode, new \Exception(sprintf('Exception from Dropbox: %s', $result)));
} }
curl_close($ch); throw new \Exception(sprintf('Exception from Dropbox: %s', $result));
}
return $postOptions['response_body_is_json'] return $postOptions['response_body_is_json']
? json_decode($result) ? json_decode($result)
: $result; : $result;
} }
finally
{
curl_close($ch);
}
}
} }

View File

@ -21,6 +21,7 @@ return [
'dropbox' => [ 'dropbox' => [
'authorise_url' => 'https://www.dropbox.com/oauth2/authorize', 'authorise_url' => 'https://www.dropbox.com/oauth2/authorize',
'delete_url' => 'https://api.dropboxapi.com/2/files/delete_v2',
'download_url' => 'https://content.dropboxapi.com/2/files/download', 'download_url' => 'https://content.dropboxapi.com/2/files/download',
'token_url' => 'https://api.dropbox.com/oauth2/token', 'token_url' => 'https://api.dropbox.com/oauth2/token',
'upload_url' => 'https://content.dropboxapi.com/2/files/upload' 'upload_url' => 'https://content.dropboxapi.com/2/files/upload'

View File

@ -80,6 +80,14 @@ class PermissionsSeeder extends Seeder
'is_default' => false, 'is_default' => false,
'sort_order' => 0 'sort_order' => 0
]); ]);
// admin:manage-services = controls if external servies can be managed
DatabaseSeeder::createOrUpdate('permissions', [
'section' => 'admin',
'description' => 'manage-services',
'is_default' => false,
'sort_order' => 0
]);
} }
private function seedAlbumPermissions() private function seedAlbumPermissions()

View File

@ -105,6 +105,12 @@ function AnalyseAlbumViewModel() {
}); });
item.isSuccessful = false; item.isSuccessful = false;
item.isPending = false; item.isPending = false;
var indexToRemove = self.imagesInProgress.indexOf(item);
if (indexToRemove > -1)
{
self.imagesInProgress.splice(indexToRemove, 1);
}
} }
} }
} }

View File

@ -7,6 +7,7 @@ return [
'manage-comments' => 'Manage comments', 'manage-comments' => 'Manage comments',
'manage-groups' => 'Manage user groups', 'manage-groups' => 'Manage user groups',
'manage-labels' => 'Manage photo labels', 'manage-labels' => 'Manage photo labels',
'manage-services' => 'Manage external services',
'manage-storage' => 'Manage storage locations', 'manage-storage' => 'Manage storage locations',
'manage-users' => 'Manage users' 'manage-users' => 'Manage users'
], ],

View File

@ -23,7 +23,7 @@
<p class="text-success"><span v-text="image.name"></span> ... <i class="fa fa-fw fa-check"></i></p> <p class="text-success"><span v-text="image.name"></span> ... <i class="fa fa-fw fa-check"></i></p>
</div> </div>
<div v-for="image in imagesInProgress.slice(0, 5)" style="margin-top: 20px;"> <div v-for="image in imagesInProgress.slice(0, 5)" style="margin-top: 20px;">
<p><span v-text="image.name"></span> ... <i class="fa fa-fw fa-refresh"></i></p> <p><span v-text="image.name"></span> ... <i class="fa fa-fw fa-sync"></i></p>
</div> </div>
<div v-if="imagesInProgress.length > 5"> <div v-if="imagesInProgress.length > 5">

View File

@ -102,6 +102,10 @@
@include(Theme::viewName('partials.permission_checkbox'), [ @include(Theme::viewName('partials.permission_checkbox'), [
'permission' => Theme::getPermission($all_permissions, 'admin', 'manage-storage') 'permission' => Theme::getPermission($all_permissions, 'admin', 'manage-storage')
]) ])
@include(Theme::viewName('partials.permission_checkbox'), [
'permission' => Theme::getPermission($all_permissions, 'admin', 'manage-services')
])
</div> </div>
</div> </div>
</div> </div>

View File

@ -9,14 +9,14 @@
@if (!Auth::guest() && UserConfig::get('social_user_feeds')) @if (!Auth::guest() && UserConfig::get('social_user_feeds'))
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{ route('userActivityFeed') }}"> <a class="nav-link" href="{{ route('userActivityFeed') }}">
<i class="fa fa-rss"></i> @lang('navigation.navbar.activity_feed') <i class="fa fa-rss mr-1"></i> @lang('navigation.navbar.activity_feed')
</a> </a>
</li> </li>
@endif @endif
@can('photo.quick_upload') @can('photo.quick_upload')
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="#" data-toggle="modal" data-target="#quick-upload-modal"> <a class="nav-link" href="#" data-toggle="modal" data-target="#quick-upload-modal">
<i class="fa fa-plus"></i> @lang('navigation.navbar.quick_post') <i class="fa fa-plus mr-1"></i> @lang('navigation.navbar.quick_post')
</a> </a>
</li> </li>
@endcan @endcan
@ -24,7 +24,7 @@
@if (count($g_albums) > 0) @if (count($g_albums) > 0)
<li class="nav-item dropdown ml-2"> <li class="nav-item dropdown ml-2">
<a class="nav-link dropdown-toggle" href="{{ url('/') }}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <a class="nav-link dropdown-toggle" href="{{ url('/') }}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
<i class="fa fa-book"></i> @lang('navigation.navbar.albums') <i class="fa fa-book mr-1"></i> @lang('navigation.navbar.albums')
</a> </a>
<div class="dropdown-menu"> <div class="dropdown-menu">
@foreach ($g_albums_menu as $album) @foreach ($g_albums_menu as $album)
@ -41,7 +41,7 @@
@if (count($g_labels) > 0) @if (count($g_labels) > 0)
<li class="nav-item dropdown ml-2"> <li class="nav-item dropdown ml-2">
<a class="nav-link dropdown-toggle" href="{{ url('/') }}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <a class="nav-link dropdown-toggle" href="{{ url('/') }}" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
<i class="fa fa-tags"></i> @lang('navigation.navbar.labels') <i class="fa fa-tags mr-1"></i> @lang('navigation.navbar.labels')
</a> </a>
<div class="dropdown-menu"> <div class="dropdown-menu">
@foreach ($g_labels as $label) @foreach ($g_labels as $label)
@ -57,7 +57,7 @@
@if (count($g_albums) > 0 && \App\User::currentOrAnonymous()->can('statistics.public-access')) @if (count($g_albums) > 0 && \App\User::currentOrAnonymous()->can('statistics.public-access'))
<li class="nav-item ml-2"> <li class="nav-item ml-2">
<a class="nav-link" href="{{ route('statistics.index') }}"><i class="fa fa-chart-line"></i> @lang('navigation.navbar.statistics')</a> <a class="nav-link" href="{{ route('statistics.index') }}"><i class="fa fa-chart-line mr-1"></i> @lang('navigation.navbar.statistics')</a>
</li> </li>
@endif @endif