aheathershaw/blue-twilight
1
0
Fork 0
Code Issues 32 Pull Requests Releases 15 Wiki Activity

Pull 135: Backblaze B2 storage driver #138

Manually merged
aheathershaw merged 8 commits from feature/135-backblaze-driver into master 2019-09-14 15:38:16 +01:00
Conversation 0 Commits 8 Files Changed 21 +228 -19
3 changed files with 228 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a6825bcef9 - Show all commits

20
app/AlbumSources/BackblazeB2Source.php
View File

@ -74,14 +74,22 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
*/
public function fetchPhotoContent(Photo $photo, $thumbnail = null)
{
// Use the same URLs that the public would use to fetch the file
$urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail);
$pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
$ch = curl_init($urlToPhoto);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$fileContent = curl_exec($ch);
// First we need the file ID
return EntityBody::fromString($fileContent);
/** @var BackblazeB2FileIdCache $b2Cache */
$b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
if (is_null($b2Cache))
{
// TODO: lookup the file on B2 to get the file ID
Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage));
return EntityBody::fromString('');
}
return EntityBody::fromString(
$this->getClient()->downloadFile($b2Cache->b2_file_id)
);
}
/**

84
app/Services/BackblazeB2Service.php
View File

@ -55,6 +55,18 @@ class BackblazeB2Service
*/
private $config;
/**
* Current file upload token.
* @var string
*/
private $uploadAuthToken;
/**
* Current upload URL.
* @var string
*/
private $uploadUrl;
public function __construct()
{
$this->config = config('services.backblaze_b2');
@ -92,15 +104,13 @@ class BackblazeB2Service
public function downloadFile($fileID)
{
$downloadToken = $this->getDownloadAuthToken();
return $this->sendRequest(
sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)),
sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID)),
'GET',
null,
[
'http_headers' => [
sprintf('Authorization: %s', $downloadToken)
sprintf('Authorization: %s', $this->authToken)
],
'response_body_is_json' => false
]
@ -155,6 +165,35 @@ class BackblazeB2Service
throw new \Exception('No upload URL/authorization token returned from Backblaze B2.');
}
$exponentialBackoff = 1;
$numberOfRetries = 5; // this effectively gives us 31 seconds of retries (1+2+4+8+16)
$numberOfTimesTried = 0;
while ($numberOfTimesTried < $numberOfRetries)
{
try
{
return $this->uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken);
}
catch (BackblazeRetryException $ex)
{
sleep($exponentialBackoff);
// Get a new upload token
$this->uploadAuthToken = null;
$this->uploadUrl = null;
list($uploadUrl, $authorizationToken) = $this->getUploadUrl();
// Keep backing off
$exponentialBackoff *= $exponentialBackoff;
$numberOfTimesTried++;
}
}
}
private function uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken)
{
$fileSize = filesize($pathToFileToUpload);
$handle = fopen($pathToFileToUpload, 'r');
$fileContents = fread($handle, $fileSize);
@ -168,7 +207,7 @@ class BackblazeB2Service
sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage))
];
$result = $this->sendRequest(
$result = $this->sendRequestReal(
$uploadUrl,
'POST',
$fileContents,
@ -200,15 +239,21 @@ class BackblazeB2Service
throw new \Exception(sprintf('The bucket \'%s\' was not found or your API key does not have access.', $bucketName));
}
private function getUploadUrl()
private function getUploadUrl($alwaysGetNewToken = false)
{
$result = $this->sendRequest(
sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl),
'POST',
['bucketId' => $this->bucketId]
);
if (is_null($this->uploadAuthToken) || $alwaysGetNewToken)
{
$result = $this->sendRequest(
sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl),
'POST',
['bucketId' => $this->bucketId]
);
return [$result->uploadUrl, $result->authorizationToken];
$this->uploadAuthToken = $result->authorizationToken;
$this->uploadUrl = $result->uploadUrl;
}
return [$this->uploadUrl, $this->uploadAuthToken];
}
private function getBasicHttpClient($url, $method = 'GET', array $httpHeaders = [])
@ -252,7 +297,15 @@ class BackblazeB2Service
}
catch (BackblazeRetryException $ex)
{
// Clear the upload token if requested
if (isset($postOptions['clear_upload_token_on_retry']) && $postOptions['clear_upload_token_on_retry'])
{
$this->uploadAuthToken = null;
$this->uploadUrl = null;
}
// Keep backing off
sleep($exponentialBackoff);
$exponentialBackoff *= $exponentialBackoff;
$numberOfTimesTried++;
}
@ -318,7 +371,12 @@ class BackblazeB2Service
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
Log::info(sprintf('Received HTTP code %d', $httpCode));
Log::debug($result);
// Only log a result if we have one and it's in JSON format (i.e. not a file download)
if (!is_null($result) && $result !== false && $postOptions['response_body_is_json'])
{
Log::debug($result);
}
// According to the Backblaze B2 Protocol, if we get a 500/503, we should retry the request
if ($httpCode == 500 || $httpCode == 503)

143
public/b2_test.php Normal file
View File

@ -0,0 +1,143 @@
<?php
function b2_authorize_account()
{
$application_key_id = "0023254ec9bda08000000000a"; // Obtained from your B2 account page
$application_key = "K002eARNPUlxdj1XaVJbwEYPMz0c7e8"; // Obtained from your B2 account page
$credentials = base64_encode($application_key_id . ":" . $application_key);
$url = "https://api.backblazeb2.com/b2api/v2/b2_authorize_account";
$session = curl_init($url);
// Add headers
$headers = array();
$headers[] = "Accept: application/json";
$headers[] = "Authorization: Basic " . $credentials;
curl_setopt($session, CURLOPT_HTTPHEADER, $headers); // Add headers
curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
$server_output = curl_exec($session);
curl_close ($session);
echo ($server_output);
return json_decode($server_output);
}
function b2_download_file_by_id($download_url, $auth_token)
{
//$download_url = ""; // From b2_authorize_account call
$file_id = "4_z731245f41efc196b6dda0018_f116729ca6de74b38_d20190910_m132847_c002_v0001127_t0021"; // The ID of the file you want to download
$uri = $download_url . "/b2api/v2/b2_download_file_by_id?fileId=" . $file_id;
$session = curl_init($uri);
curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
echo '<p>' . $uri . '</p>';
$server_output = curl_exec($session); // Let's do this!
if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
{
echo '<p>' . $server_output . '</p>';
}
else
{
echo '<p>' . (strlen($server_output) . ' bytes received') . '</p>'; // Tell me about the rabbits, George!
}
curl_close ($session); // Clean up
//$download_url = ""; // From b2_authorize_account call
$file_id = "4_z731245f41efc196b6dda0018_f116729ca6de74b38_d20190910_m132847_c002_v0001127_t0021"; // The ID of the file you want to download
$uri = $download_url . "/b2api/v2/b2_download_file_by_id?fileId=" . $file_id . '&Authorization=' . $auth_token;
$session = curl_init($uri);
curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
echo '<p>' . $uri . '</p>';
$server_output = curl_exec($session); // Let's do this!
if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
{
echo '<p>' . $server_output . '</p>';
}
else
{
echo '<p>' . (strlen($server_output) . ' bytes received') . '</p>'; // Tell me about the rabbits, George!
}
curl_close ($session); // Clean up
}
function b2_download_file_by_name($download_url, $auth_token)
{
//$download_url = ""; // From b2_authorize_account call
$bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from
$file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download
$uri = $download_url . "/file/" . $bucket_name . "/" . $file_name;
$session = curl_init($uri);
curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
curl_setopt($session, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
echo '<p>' . $uri . '</p>';
$server_output = curl_exec($session); // Let's do this!
if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
{
echo '<p>' . $server_output . '</p>';
}
else
{
echo '<p>' . (strlen($server_output) . ' bytes received') . '</p>'; // Tell me about the rabbits, George!
}
curl_close ($session); // Clean up
// You will need to use the account authorization token if your bucket's type is allPrivate.
//$download_url = ""; // From b2_authorize_account call
$bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from
$file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download
//$auth_token = ""; // From b2_authorize_account call
$uri = $download_url . "/file/" . $bucket_name . "/" . $file_name . '?Authorization=' . $auth_token;
$session = curl_init($uri);
curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP POST
curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
echo '<p>' . $uri . '</p>';
$server_output = curl_exec($session); // Let's do this!
if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
{
echo '<p>' . $server_output . '</p>';
}
else
{
echo '<p>' . (strlen($server_output) . ' bytes received') . '</p>'; // Tell me about the rabbits, George!
}
curl_close ($session); // Clean up
}
?>
<h2>b2_authorize_account</h2>
<?php $authorize_account_result = b2_authorize_account(); ?>
<h2>b2_download_file_by_name</h2>
<?php b2_download_file_by_name($authorize_account_result->downloadUrl, $authorize_account_result->authorizationToken); ?>
<h2>b2_download_file_by_id</h2>
<?php b2_download_file_by_id($authorize_account_result->downloadUrl, $authorize_account_result->authorizationToken); ?>