Pull 135: Backblaze B2 storage driver #138

Manually merged
aheathershaw merged 8 commits from feature/135-backblaze-driver into master 2019-09-14 15:38:16 +01:00
6 changed files with 220 additions and 29 deletions
Showing only changes of commit ce03b2596f - Show all commits

View File

@ -2,6 +2,7 @@
namespace App;
use App\AlbumSources\AlbumSourceBase;
use App\AlbumSources\IAlbumSource;
use App\AlbumSources\LocalFilesystemSource;
use App\Helpers\MiscHelper;
@ -158,10 +159,7 @@ class Album extends Model
*/
public function getAlbumSource()
{
$fullClassName = sprintf('App\AlbumSources\%s', $this->storage->source);
/** @var IAlbumSource $source */
$source = new $fullClassName;
$source = AlbumSourceBase::make($this->storage->source);
$source->setAlbum($this);
$source->setConfiguration($this->storage);

View File

@ -17,6 +17,31 @@ abstract class AlbumSourceBase
*/
protected $configuration;
/**
* @var mixed
*/
private static $albumSourceCache = [];
/**
* Makes an album source class for the given source name (relative class name.)
* @param string $sourceName Name of the source.
* @return IAlbumSource
*/
public static function make($sourceName)
{
$fullClassName = sprintf('App\AlbumSources\%s', $sourceName);
if (!array_key_exists($fullClassName, self::$albumSourceCache))
{
/** @var IAlbumSource $source */
$source = app($fullClassName);
self::$albumSourceCache[$fullClassName] = $source;
}
return self::$albumSourceCache[$fullClassName];
}
public function setAlbum(Album $album)
{
$this->album = $album;

View File

@ -2,10 +2,12 @@
namespace App\AlbumSources;
use App\BackblazeB2FileIdCache;
use App\Photo;
use App\Services\BackblazeB2Service;
use App\Storage;
use Guzzle\Http\EntityBody;
use Illuminate\Support\Facades\Log;
class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
{
@ -36,7 +38,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
*/
public function deleteAlbumContents()
{
// TODO: Implement deleteAlbumContents() method.
// No need to do anything for the album container - once the files are gone, the virtual folder is also gone
}
/**
@ -47,7 +49,21 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
*/
public function deleteThumbnail(Photo $photo, $thumbnail = null)
{
// TODO: Implement deleteThumbnail() method.
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
// Create or update our cache record
/** @var BackblazeB2FileIdCache $b2Cache */
$b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
if (is_null($b2Cache))
{
// TODO: lookup the file on B2 to get the file ID
Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage));
return;
}
$this->getClient()->deleteFile($b2Cache->b2_file_id, $pathOnStorage);
$b2Cache->delete();
}
/**
@ -58,7 +74,14 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
*/
public function fetchPhotoContent(Photo $photo, $thumbnail = null)
{
// TODO: Implement fetchPhotoContent() method.
// Use the same URLs that the public would use to fetch the file
$urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail);
$ch = curl_init($urlToPhoto);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$fileContent = curl_exec($ch);
return EntityBody::fromString($fileContent);
}
/**
@ -93,6 +116,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
switch ($this->bucketType)
{
case self::BUCKET_TYPE_PRIVATE:
// TODO: use the B2 b2_download_file_by_id method so filenames are harder to guess
if (is_null($this->downloadToken))
{
$this->downloadToken = $client->getDownloadAuthToken();
@ -116,7 +140,25 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
{
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
$this->getClient()->uploadFile($tempFilename, $pathOnStorage);
// Upload the file to B2
$b2FileID = $this->getClient()->uploadFile($tempFilename, $pathOnStorage);
// Create or update our cache record
$b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
if (is_null($b2Cache))
{
$b2Cache = new BackblazeB2FileIdCache([
'photo_id' => $photo->id,
'storage_path' => $pathOnStorage,
'b2_file_id' => $b2FileID
]);
}
else
{
$b2Cache->b2_file_id = $b2FileID;
}
$b2Cache->save();
}
public function setConfiguration(Storage $configuration)

View File

@ -0,0 +1,19 @@
<?php
namespace App;
use Illuminate\Database\Eloquent\Model;
class BackblazeB2FileIdCache extends Model
{
/**
* The attributes that are mass assignable.
*
* @var array
*/
protected $fillable = [
'photo_id',
'storage_path',
'b2_file_id'
];
}

View File

@ -2,6 +2,8 @@
namespace App\Services;
use Illuminate\Support\Facades\Log;
class BackblazeB2Service
{
/**
@ -75,6 +77,35 @@ class BackblazeB2Service
}
}
public function deleteFile($fileID, $fileName)
{
$this->sendRequest(
sprintf('%s/b2api/v2/b2_delete_file_version', $this->accountApiUrl),
'POST',
[
'fileId' => $fileID,
'fileName' => $fileName
]
);
}
public function downloadFile($fileID)
{
$downloadToken = $this->getDownloadAuthToken();
return $this->sendRequest(
sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)),
'GET',
null,
[
'http_headers' => [
sprintf('Authorization: %s', $downloadToken)
],
'response_body_is_json' => false
]
);
}
public function getBucketType()
{
return $this->bucketType;
@ -129,24 +160,24 @@ class BackblazeB2Service
fclose($handle);
$fileContentsSha1 = sha1_file($pathToFileToUpload);
$ch = $this->getBasicHttpClient($uploadUrl, 'POST', [
$httpHeaders = [
sprintf('Authorization: %s', $authorizationToken),
'Content-Type: b2/x-auto',
sprintf('X-Bz-Content-Sha1: %s', $fileContentsSha1),
sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage))
]);
];
curl_setopt($ch, CURLOPT_POSTFIELDS, $fileContents);
$result = $this->sendRequest(
$uploadUrl,
'POST',
$fileContents,
[
'http_headers' => $httpHeaders,
'post_body_is_json' => false
]
);
$result = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($httpCode != 200 && $httpCode != 304)
{
throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result));
}
curl_close($ch);
return $result->fileId;
}
private function getBucketDetailsFromName($bucketName)
@ -206,31 +237,67 @@ class BackblazeB2Service
return $ch;
}
private function sendRequest($url, $method = 'GET', $postData = null)
private function sendRequest($url, $method = 'GET', $postData = null, array $postOptions = [])
{
$httpHeaders = [];
$postOptions = array_merge(
[
'authorization_token' => null,
'http_headers' => [],
'post_body_is_json' => true,
'response_body_is_json' => true
],
$postOptions
);
$httpHeaders = $postOptions['http_headers'];
if (is_null($this->authToken))
// Some methods may need to override the authorization token used
if (empty($postOptions['authorization_token']))
{
// No auth token yet, use username/password
$httpHeaders[] = sprintf('Authorization: Basic %s', base64_encode($this->authHeader));
// No override - work out which auth token to use
if (is_null($this->authToken))
{
// No auth token yet, use username/password
$httpHeaders[] = sprintf('Authorization: Basic %s', base64_encode($this->authHeader));
}
else
{
// Use the auth token we have
$httpHeaders[] = sprintf('Authorization: %s', $this->authToken);
}
}
else
{
// Use the auth token we have
$httpHeaders[] = sprintf('Authorization: %s', $this->authToken);
// Override - use the auth token specified
$httpHeaders[] = sprintf('Authorization: %s', $postOptions['authorization_token']);
}
$ch = $this->getBasicHttpClient($url, $method, $httpHeaders);
if (!is_null($postData))
{
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData));
if ($postOptions['post_body_is_json'])
{
$postData = json_encode($postData);
}
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
}
Log::info(sprintf('%s: %s', strtoupper($method), $url));
Log::debug('HTTP headers:', $httpHeaders);
// Only log a post body if we have one and it's in JSON format (i.e. not a file upload)
if (!is_null($postData) && $postOptions['post_body_is_json'])
{
Log::debug($postData);
}
$result = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
Log::info(sprintf('Received HTTP code %d', $httpCode));
Log::debug($result);
if ($httpCode != 200 && $httpCode != 304)
{
throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result));
@ -238,6 +305,8 @@ class BackblazeB2Service
curl_close($ch);
return json_decode($result);
return $postOptions['response_body_is_json']
? json_decode($result)
: $result;
}
}

View File

@ -0,0 +1,38 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateBackblazeB2FileIdCachesTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('backblaze_b2_file_id_caches', function (Blueprint $table) {
$table->bigIncrements('id');
$table->unsignedBigInteger('photo_id');
$table->string('storage_path');
$table->string('b2_file_id');
$table->timestamps();
$table->foreign('photo_id')
->references('id')->on('photos')
->onDelete('cascade');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('backblaze_b2_file_id_caches');
}
}