From ce03b2596ff18d187de565cb649b63a11c485ef8 Mon Sep 17 00:00:00 2001 From: Andy Heathershaw Date: Tue, 10 Sep 2019 15:11:53 +0100 Subject: [PATCH] Backblaze #135 - album storage driver is now cached to maintain state within the same request, prevents multiple calls to B2. Images can now be deleted and (I think) edited. --- app/Album.php | 6 +- app/AlbumSources/AlbumSourceBase.php | 25 ++++ app/AlbumSources/BackblazeB2Source.php | 50 +++++++- app/BackblazeB2FileIdCache.php | 19 +++ app/Services/BackblazeB2Service.php | 111 ++++++++++++++---- ...eate_backblaze_b2_file_id_caches_table.php | 38 ++++++ 6 files changed, 220 insertions(+), 29 deletions(-) create mode 100644 app/BackblazeB2FileIdCache.php create mode 100644 database/migrations/2019_09_10_085020_create_backblaze_b2_file_id_caches_table.php diff --git a/app/Album.php b/app/Album.php index e375df9..158357c 100644 --- a/app/Album.php +++ b/app/Album.php @@ -2,6 +2,7 @@ namespace App; +use App\AlbumSources\AlbumSourceBase; use App\AlbumSources\IAlbumSource; use App\AlbumSources\LocalFilesystemSource; use App\Helpers\MiscHelper; @@ -158,10 +159,7 @@ class Album extends Model */ public function getAlbumSource() { - $fullClassName = sprintf('App\AlbumSources\%s', $this->storage->source); - - /** @var IAlbumSource $source */ - $source = new $fullClassName; + $source = AlbumSourceBase::make($this->storage->source); $source->setAlbum($this); $source->setConfiguration($this->storage); diff --git a/app/AlbumSources/AlbumSourceBase.php b/app/AlbumSources/AlbumSourceBase.php index 364bc39..374e17f 100644 --- a/app/AlbumSources/AlbumSourceBase.php +++ b/app/AlbumSources/AlbumSourceBase.php @@ -17,6 +17,31 @@ abstract class AlbumSourceBase */ protected $configuration; + /** + * @var mixed + */ + private static $albumSourceCache = []; + + /** + * Makes an album source class for the given source name (relative class name.) + * @param string $sourceName Name of the source. + * @return IAlbumSource + */ + public static function make($sourceName) + { + $fullClassName = sprintf('App\AlbumSources\%s', $sourceName); + + if (!array_key_exists($fullClassName, self::$albumSourceCache)) + { + /** @var IAlbumSource $source */ + $source = app($fullClassName); + + self::$albumSourceCache[$fullClassName] = $source; + } + + return self::$albumSourceCache[$fullClassName]; + } + public function setAlbum(Album $album) { $this->album = $album; diff --git a/app/AlbumSources/BackblazeB2Source.php b/app/AlbumSources/BackblazeB2Source.php index c6c21d7..b7e3986 100644 --- a/app/AlbumSources/BackblazeB2Source.php +++ b/app/AlbumSources/BackblazeB2Source.php @@ -2,10 +2,12 @@ namespace App\AlbumSources; +use App\BackblazeB2FileIdCache; use App\Photo; use App\Services\BackblazeB2Service; use App\Storage; use Guzzle\Http\EntityBody; +use Illuminate\Support\Facades\Log; class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource { @@ -36,7 +38,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource */ public function deleteAlbumContents() { - // TODO: Implement deleteAlbumContents() method. + // No need to do anything for the album container - once the files are gone, the virtual folder is also gone } /** @@ -47,7 +49,21 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource */ public function deleteThumbnail(Photo $photo, $thumbnail = null) { - // TODO: Implement deleteThumbnail() method. + $pathOnStorage = $this->getPathToPhoto($photo, $thumbnail); + + // Create or update our cache record + + /** @var BackblazeB2FileIdCache $b2Cache */ + $b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first(); + if (is_null($b2Cache)) + { + // TODO: lookup the file on B2 to get the file ID + Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage)); + return; + } + + $this->getClient()->deleteFile($b2Cache->b2_file_id, $pathOnStorage); + $b2Cache->delete(); } /** @@ -58,7 +74,14 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource */ public function fetchPhotoContent(Photo $photo, $thumbnail = null) { - // TODO: Implement fetchPhotoContent() method. + // Use the same URLs that the public would use to fetch the file + $urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail); + + $ch = curl_init($urlToPhoto); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $fileContent = curl_exec($ch); + + return EntityBody::fromString($fileContent); } /** @@ -93,6 +116,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource switch ($this->bucketType) { case self::BUCKET_TYPE_PRIVATE: + // TODO: use the B2 b2_download_file_by_id method so filenames are harder to guess if (is_null($this->downloadToken)) { $this->downloadToken = $client->getDownloadAuthToken(); @@ -116,7 +140,25 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource { $pathOnStorage = $this->getPathToPhoto($photo, $thumbnail); - $this->getClient()->uploadFile($tempFilename, $pathOnStorage); + // Upload the file to B2 + $b2FileID = $this->getClient()->uploadFile($tempFilename, $pathOnStorage); + + // Create or update our cache record + $b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first(); + if (is_null($b2Cache)) + { + $b2Cache = new BackblazeB2FileIdCache([ + 'photo_id' => $photo->id, + 'storage_path' => $pathOnStorage, + 'b2_file_id' => $b2FileID + ]); + } + else + { + $b2Cache->b2_file_id = $b2FileID; + } + + $b2Cache->save(); } public function setConfiguration(Storage $configuration) diff --git a/app/BackblazeB2FileIdCache.php b/app/BackblazeB2FileIdCache.php new file mode 100644 index 0000000..b1765cd --- /dev/null +++ b/app/BackblazeB2FileIdCache.php @@ -0,0 +1,19 @@ +sendRequest( + sprintf('%s/b2api/v2/b2_delete_file_version', $this->accountApiUrl), + 'POST', + [ + 'fileId' => $fileID, + 'fileName' => $fileName + ] + ); + } + + public function downloadFile($fileID) + { + $downloadToken = $this->getDownloadAuthToken(); + + return $this->sendRequest( + sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)), + 'GET', + null, + [ + 'http_headers' => [ + sprintf('Authorization: %s', $downloadToken) + ], + 'response_body_is_json' => false + ] + ); + } + public function getBucketType() { return $this->bucketType; @@ -129,24 +160,24 @@ class BackblazeB2Service fclose($handle); $fileContentsSha1 = sha1_file($pathToFileToUpload); - $ch = $this->getBasicHttpClient($uploadUrl, 'POST', [ + $httpHeaders = [ sprintf('Authorization: %s', $authorizationToken), 'Content-Type: b2/x-auto', sprintf('X-Bz-Content-Sha1: %s', $fileContentsSha1), sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage)) - ]); + ]; - curl_setopt($ch, CURLOPT_POSTFIELDS, $fileContents); + $result = $this->sendRequest( + $uploadUrl, + 'POST', + $fileContents, + [ + 'http_headers' => $httpHeaders, + 'post_body_is_json' => false + ] + ); - $result = curl_exec($ch); - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - - if ($httpCode != 200 && $httpCode != 304) - { - throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result)); - } - - curl_close($ch); + return $result->fileId; } private function getBucketDetailsFromName($bucketName) @@ -206,31 +237,67 @@ class BackblazeB2Service return $ch; } - private function sendRequest($url, $method = 'GET', $postData = null) + private function sendRequest($url, $method = 'GET', $postData = null, array $postOptions = []) { - $httpHeaders = []; + $postOptions = array_merge( + [ + 'authorization_token' => null, + 'http_headers' => [], + 'post_body_is_json' => true, + 'response_body_is_json' => true + ], + $postOptions + ); + $httpHeaders = $postOptions['http_headers']; - if (is_null($this->authToken)) + // Some methods may need to override the authorization token used + if (empty($postOptions['authorization_token'])) { - // No auth token yet, use username/password - $httpHeaders[] = sprintf('Authorization: Basic %s', base64_encode($this->authHeader)); + // No override - work out which auth token to use + if (is_null($this->authToken)) + { + // No auth token yet, use username/password + $httpHeaders[] = sprintf('Authorization: Basic %s', base64_encode($this->authHeader)); + } + else + { + // Use the auth token we have + $httpHeaders[] = sprintf('Authorization: %s', $this->authToken); + } } else { - // Use the auth token we have - $httpHeaders[] = sprintf('Authorization: %s', $this->authToken); + // Override - use the auth token specified + $httpHeaders[] = sprintf('Authorization: %s', $postOptions['authorization_token']); } $ch = $this->getBasicHttpClient($url, $method, $httpHeaders); if (!is_null($postData)) { - curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData)); + if ($postOptions['post_body_is_json']) + { + $postData = json_encode($postData); + } + + curl_setopt($ch, CURLOPT_POSTFIELDS, $postData); + } + + Log::info(sprintf('%s: %s', strtoupper($method), $url)); + Log::debug('HTTP headers:', $httpHeaders); + + // Only log a post body if we have one and it's in JSON format (i.e. not a file upload) + if (!is_null($postData) && $postOptions['post_body_is_json']) + { + Log::debug($postData); } $result = curl_exec($ch); $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + Log::info(sprintf('Received HTTP code %d', $httpCode)); + Log::debug($result); + if ($httpCode != 200 && $httpCode != 304) { throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result)); @@ -238,6 +305,8 @@ class BackblazeB2Service curl_close($ch); - return json_decode($result); + return $postOptions['response_body_is_json'] + ? json_decode($result) + : $result; } } \ No newline at end of file diff --git a/database/migrations/2019_09_10_085020_create_backblaze_b2_file_id_caches_table.php b/database/migrations/2019_09_10_085020_create_backblaze_b2_file_id_caches_table.php new file mode 100644 index 0000000..1b7ac4d --- /dev/null +++ b/database/migrations/2019_09_10_085020_create_backblaze_b2_file_id_caches_table.php @@ -0,0 +1,38 @@ +bigIncrements('id'); + $table->unsignedBigInteger('photo_id'); + $table->string('storage_path'); + $table->string('b2_file_id'); + $table->timestamps(); + + $table->foreign('photo_id') + ->references('id')->on('photos') + ->onDelete('cascade'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('backblaze_b2_file_id_caches'); + } +}