Pull 135: Backblaze B2 storage driver #138
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
namespace App;
|
namespace App;
|
||||||
|
|
||||||
|
use App\AlbumSources\AlbumSourceBase;
|
||||||
use App\AlbumSources\IAlbumSource;
|
use App\AlbumSources\IAlbumSource;
|
||||||
use App\AlbumSources\LocalFilesystemSource;
|
use App\AlbumSources\LocalFilesystemSource;
|
||||||
use App\Helpers\MiscHelper;
|
use App\Helpers\MiscHelper;
|
||||||
@ -158,10 +159,7 @@ class Album extends Model
|
|||||||
*/
|
*/
|
||||||
public function getAlbumSource()
|
public function getAlbumSource()
|
||||||
{
|
{
|
||||||
$fullClassName = sprintf('App\AlbumSources\%s', $this->storage->source);
|
$source = AlbumSourceBase::make($this->storage->source);
|
||||||
|
|
||||||
/** @var IAlbumSource $source */
|
|
||||||
$source = new $fullClassName;
|
|
||||||
$source->setAlbum($this);
|
$source->setAlbum($this);
|
||||||
$source->setConfiguration($this->storage);
|
$source->setConfiguration($this->storage);
|
||||||
|
|
||||||
|
@ -17,6 +17,31 @@ abstract class AlbumSourceBase
|
|||||||
*/
|
*/
|
||||||
protected $configuration;
|
protected $configuration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @var mixed
|
||||||
|
*/
|
||||||
|
private static $albumSourceCache = [];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Makes an album source class for the given source name (relative class name.)
|
||||||
|
* @param string $sourceName Name of the source.
|
||||||
|
* @return IAlbumSource
|
||||||
|
*/
|
||||||
|
public static function make($sourceName)
|
||||||
|
{
|
||||||
|
$fullClassName = sprintf('App\AlbumSources\%s', $sourceName);
|
||||||
|
|
||||||
|
if (!array_key_exists($fullClassName, self::$albumSourceCache))
|
||||||
|
{
|
||||||
|
/** @var IAlbumSource $source */
|
||||||
|
$source = app($fullClassName);
|
||||||
|
|
||||||
|
self::$albumSourceCache[$fullClassName] = $source;
|
||||||
|
}
|
||||||
|
|
||||||
|
return self::$albumSourceCache[$fullClassName];
|
||||||
|
}
|
||||||
|
|
||||||
public function setAlbum(Album $album)
|
public function setAlbum(Album $album)
|
||||||
{
|
{
|
||||||
$this->album = $album;
|
$this->album = $album;
|
||||||
|
@ -2,10 +2,12 @@
|
|||||||
|
|
||||||
namespace App\AlbumSources;
|
namespace App\AlbumSources;
|
||||||
|
|
||||||
|
use App\BackblazeB2FileIdCache;
|
||||||
use App\Photo;
|
use App\Photo;
|
||||||
use App\Services\BackblazeB2Service;
|
use App\Services\BackblazeB2Service;
|
||||||
use App\Storage;
|
use App\Storage;
|
||||||
use Guzzle\Http\EntityBody;
|
use Guzzle\Http\EntityBody;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
||||||
{
|
{
|
||||||
@ -36,7 +38,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
|||||||
*/
|
*/
|
||||||
public function deleteAlbumContents()
|
public function deleteAlbumContents()
|
||||||
{
|
{
|
||||||
// TODO: Implement deleteAlbumContents() method.
|
// No need to do anything for the album container - once the files are gone, the virtual folder is also gone
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -47,7 +49,21 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
|||||||
*/
|
*/
|
||||||
public function deleteThumbnail(Photo $photo, $thumbnail = null)
|
public function deleteThumbnail(Photo $photo, $thumbnail = null)
|
||||||
{
|
{
|
||||||
// TODO: Implement deleteThumbnail() method.
|
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
|
||||||
|
|
||||||
|
// Create or update our cache record
|
||||||
|
|
||||||
|
/** @var BackblazeB2FileIdCache $b2Cache */
|
||||||
|
$b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
|
||||||
|
if (is_null($b2Cache))
|
||||||
|
{
|
||||||
|
// TODO: lookup the file on B2 to get the file ID
|
||||||
|
Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->getClient()->deleteFile($b2Cache->b2_file_id, $pathOnStorage);
|
||||||
|
$b2Cache->delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -58,7 +74,14 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
|||||||
*/
|
*/
|
||||||
public function fetchPhotoContent(Photo $photo, $thumbnail = null)
|
public function fetchPhotoContent(Photo $photo, $thumbnail = null)
|
||||||
{
|
{
|
||||||
// TODO: Implement fetchPhotoContent() method.
|
// Use the same URLs that the public would use to fetch the file
|
||||||
|
$urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail);
|
||||||
|
|
||||||
|
$ch = curl_init($urlToPhoto);
|
||||||
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||||
|
$fileContent = curl_exec($ch);
|
||||||
|
|
||||||
|
return EntityBody::fromString($fileContent);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -93,6 +116,7 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
|||||||
switch ($this->bucketType)
|
switch ($this->bucketType)
|
||||||
{
|
{
|
||||||
case self::BUCKET_TYPE_PRIVATE:
|
case self::BUCKET_TYPE_PRIVATE:
|
||||||
|
// TODO: use the B2 b2_download_file_by_id method so filenames are harder to guess
|
||||||
if (is_null($this->downloadToken))
|
if (is_null($this->downloadToken))
|
||||||
{
|
{
|
||||||
$this->downloadToken = $client->getDownloadAuthToken();
|
$this->downloadToken = $client->getDownloadAuthToken();
|
||||||
@ -116,7 +140,25 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
|
|||||||
{
|
{
|
||||||
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
|
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
|
||||||
|
|
||||||
$this->getClient()->uploadFile($tempFilename, $pathOnStorage);
|
// Upload the file to B2
|
||||||
|
$b2FileID = $this->getClient()->uploadFile($tempFilename, $pathOnStorage);
|
||||||
|
|
||||||
|
// Create or update our cache record
|
||||||
|
$b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
|
||||||
|
if (is_null($b2Cache))
|
||||||
|
{
|
||||||
|
$b2Cache = new BackblazeB2FileIdCache([
|
||||||
|
'photo_id' => $photo->id,
|
||||||
|
'storage_path' => $pathOnStorage,
|
||||||
|
'b2_file_id' => $b2FileID
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$b2Cache->b2_file_id = $b2FileID;
|
||||||
|
}
|
||||||
|
|
||||||
|
$b2Cache->save();
|
||||||
}
|
}
|
||||||
|
|
||||||
public function setConfiguration(Storage $configuration)
|
public function setConfiguration(Storage $configuration)
|
||||||
|
19
app/BackblazeB2FileIdCache.php
Normal file
19
app/BackblazeB2FileIdCache.php
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App;
|
||||||
|
|
||||||
|
use Illuminate\Database\Eloquent\Model;
|
||||||
|
|
||||||
|
class BackblazeB2FileIdCache extends Model
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* The attributes that are mass assignable.
|
||||||
|
*
|
||||||
|
* @var array
|
||||||
|
*/
|
||||||
|
protected $fillable = [
|
||||||
|
'photo_id',
|
||||||
|
'storage_path',
|
||||||
|
'b2_file_id'
|
||||||
|
];
|
||||||
|
}
|
@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
namespace App\Services;
|
namespace App\Services;
|
||||||
|
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
class BackblazeB2Service
|
class BackblazeB2Service
|
||||||
{
|
{
|
||||||
/**
|
/**
|
||||||
@ -75,6 +77,35 @@ class BackblazeB2Service
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function deleteFile($fileID, $fileName)
|
||||||
|
{
|
||||||
|
$this->sendRequest(
|
||||||
|
sprintf('%s/b2api/v2/b2_delete_file_version', $this->accountApiUrl),
|
||||||
|
'POST',
|
||||||
|
[
|
||||||
|
'fileId' => $fileID,
|
||||||
|
'fileName' => $fileName
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function downloadFile($fileID)
|
||||||
|
{
|
||||||
|
$downloadToken = $this->getDownloadAuthToken();
|
||||||
|
|
||||||
|
return $this->sendRequest(
|
||||||
|
sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)),
|
||||||
|
'GET',
|
||||||
|
null,
|
||||||
|
[
|
||||||
|
'http_headers' => [
|
||||||
|
sprintf('Authorization: %s', $downloadToken)
|
||||||
|
],
|
||||||
|
'response_body_is_json' => false
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
public function getBucketType()
|
public function getBucketType()
|
||||||
{
|
{
|
||||||
return $this->bucketType;
|
return $this->bucketType;
|
||||||
@ -129,24 +160,24 @@ class BackblazeB2Service
|
|||||||
fclose($handle);
|
fclose($handle);
|
||||||
$fileContentsSha1 = sha1_file($pathToFileToUpload);
|
$fileContentsSha1 = sha1_file($pathToFileToUpload);
|
||||||
|
|
||||||
$ch = $this->getBasicHttpClient($uploadUrl, 'POST', [
|
$httpHeaders = [
|
||||||
sprintf('Authorization: %s', $authorizationToken),
|
sprintf('Authorization: %s', $authorizationToken),
|
||||||
'Content-Type: b2/x-auto',
|
'Content-Type: b2/x-auto',
|
||||||
sprintf('X-Bz-Content-Sha1: %s', $fileContentsSha1),
|
sprintf('X-Bz-Content-Sha1: %s', $fileContentsSha1),
|
||||||
sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage))
|
sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage))
|
||||||
]);
|
];
|
||||||
|
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, $fileContents);
|
$result = $this->sendRequest(
|
||||||
|
$uploadUrl,
|
||||||
|
'POST',
|
||||||
|
$fileContents,
|
||||||
|
[
|
||||||
|
'http_headers' => $httpHeaders,
|
||||||
|
'post_body_is_json' => false
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
$result = curl_exec($ch);
|
return $result->fileId;
|
||||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
|
||||||
|
|
||||||
if ($httpCode != 200 && $httpCode != 304)
|
|
||||||
{
|
|
||||||
throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result));
|
|
||||||
}
|
|
||||||
|
|
||||||
curl_close($ch);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function getBucketDetailsFromName($bucketName)
|
private function getBucketDetailsFromName($bucketName)
|
||||||
@ -206,10 +237,23 @@ class BackblazeB2Service
|
|||||||
return $ch;
|
return $ch;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function sendRequest($url, $method = 'GET', $postData = null)
|
private function sendRequest($url, $method = 'GET', $postData = null, array $postOptions = [])
|
||||||
{
|
{
|
||||||
$httpHeaders = [];
|
$postOptions = array_merge(
|
||||||
|
[
|
||||||
|
'authorization_token' => null,
|
||||||
|
'http_headers' => [],
|
||||||
|
'post_body_is_json' => true,
|
||||||
|
'response_body_is_json' => true
|
||||||
|
],
|
||||||
|
$postOptions
|
||||||
|
);
|
||||||
|
$httpHeaders = $postOptions['http_headers'];
|
||||||
|
|
||||||
|
// Some methods may need to override the authorization token used
|
||||||
|
if (empty($postOptions['authorization_token']))
|
||||||
|
{
|
||||||
|
// No override - work out which auth token to use
|
||||||
if (is_null($this->authToken))
|
if (is_null($this->authToken))
|
||||||
{
|
{
|
||||||
// No auth token yet, use username/password
|
// No auth token yet, use username/password
|
||||||
@ -220,17 +264,40 @@ class BackblazeB2Service
|
|||||||
// Use the auth token we have
|
// Use the auth token we have
|
||||||
$httpHeaders[] = sprintf('Authorization: %s', $this->authToken);
|
$httpHeaders[] = sprintf('Authorization: %s', $this->authToken);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Override - use the auth token specified
|
||||||
|
$httpHeaders[] = sprintf('Authorization: %s', $postOptions['authorization_token']);
|
||||||
|
}
|
||||||
|
|
||||||
$ch = $this->getBasicHttpClient($url, $method, $httpHeaders);
|
$ch = $this->getBasicHttpClient($url, $method, $httpHeaders);
|
||||||
|
|
||||||
if (!is_null($postData))
|
if (!is_null($postData))
|
||||||
{
|
{
|
||||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData));
|
if ($postOptions['post_body_is_json'])
|
||||||
|
{
|
||||||
|
$postData = json_encode($postData);
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
|
||||||
|
}
|
||||||
|
|
||||||
|
Log::info(sprintf('%s: %s', strtoupper($method), $url));
|
||||||
|
Log::debug('HTTP headers:', $httpHeaders);
|
||||||
|
|
||||||
|
// Only log a post body if we have one and it's in JSON format (i.e. not a file upload)
|
||||||
|
if (!is_null($postData) && $postOptions['post_body_is_json'])
|
||||||
|
{
|
||||||
|
Log::debug($postData);
|
||||||
}
|
}
|
||||||
|
|
||||||
$result = curl_exec($ch);
|
$result = curl_exec($ch);
|
||||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||||
|
|
||||||
|
Log::info(sprintf('Received HTTP code %d', $httpCode));
|
||||||
|
Log::debug($result);
|
||||||
|
|
||||||
if ($httpCode != 200 && $httpCode != 304)
|
if ($httpCode != 200 && $httpCode != 304)
|
||||||
{
|
{
|
||||||
throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result));
|
throw new \Exception(sprintf('Exception from Backblaze B2: %s', $result));
|
||||||
@ -238,6 +305,8 @@ class BackblazeB2Service
|
|||||||
|
|
||||||
curl_close($ch);
|
curl_close($ch);
|
||||||
|
|
||||||
return json_decode($result);
|
return $postOptions['response_body_is_json']
|
||||||
|
? json_decode($result)
|
||||||
|
: $result;
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -0,0 +1,38 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
|
||||||
|
class CreateBackblazeB2FileIdCachesTable extends Migration
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Run the migrations.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function up()
|
||||||
|
{
|
||||||
|
Schema::create('backblaze_b2_file_id_caches', function (Blueprint $table) {
|
||||||
|
$table->bigIncrements('id');
|
||||||
|
$table->unsignedBigInteger('photo_id');
|
||||||
|
$table->string('storage_path');
|
||||||
|
$table->string('b2_file_id');
|
||||||
|
$table->timestamps();
|
||||||
|
|
||||||
|
$table->foreign('photo_id')
|
||||||
|
->references('id')->on('photos')
|
||||||
|
->onDelete('cascade');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reverse the migrations.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function down()
|
||||||
|
{
|
||||||
|
Schema::dropIfExists('backblaze_b2_file_id_caches');
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user