From a6825bcef90bdaa7f05759047945540af6cacf20 Mon Sep 17 00:00:00 2001
From: Andy Heathershaw
Date: Sat, 14 Sep 2019 10:04:09 +0100
Subject: [PATCH] Backblaze #135 - implemented the re-use of the upload
token/URL. Fetching file contents now works by using the
b2_download_file_by_id method with an auth header.
---
app/AlbumSources/BackblazeB2Source.php | 20 ++--
app/Services/BackblazeB2Service.php | 84 ++++++++++++---
public/b2_test.php | 143 +++++++++++++++++++++++++
3 files changed, 228 insertions(+), 19 deletions(-)
create mode 100644 public/b2_test.php
diff --git a/app/AlbumSources/BackblazeB2Source.php b/app/AlbumSources/BackblazeB2Source.php
index c9a4c9e..5223d32 100644
--- a/app/AlbumSources/BackblazeB2Source.php
+++ b/app/AlbumSources/BackblazeB2Source.php
@@ -74,14 +74,22 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource
*/
public function fetchPhotoContent(Photo $photo, $thumbnail = null)
{
- // Use the same URLs that the public would use to fetch the file
- $urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail);
+ $pathOnStorage = $this->getPathToPhoto($photo, $thumbnail);
- $ch = curl_init($urlToPhoto);
- curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
- $fileContent = curl_exec($ch);
+ // First we need the file ID
- return EntityBody::fromString($fileContent);
+ /** @var BackblazeB2FileIdCache $b2Cache */
+ $b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first();
+ if (is_null($b2Cache))
+ {
+ // TODO: lookup the file on B2 to get the file ID
+ Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage));
+ return EntityBody::fromString('');
+ }
+
+ return EntityBody::fromString(
+ $this->getClient()->downloadFile($b2Cache->b2_file_id)
+ );
}
/**
diff --git a/app/Services/BackblazeB2Service.php b/app/Services/BackblazeB2Service.php
index 57149ee..ceab613 100644
--- a/app/Services/BackblazeB2Service.php
+++ b/app/Services/BackblazeB2Service.php
@@ -55,6 +55,18 @@ class BackblazeB2Service
*/
private $config;
+ /**
+ * Current file upload token.
+ * @var string
+ */
+ private $uploadAuthToken;
+
+ /**
+ * Current upload URL.
+ * @var string
+ */
+ private $uploadUrl;
+
public function __construct()
{
$this->config = config('services.backblaze_b2');
@@ -92,15 +104,13 @@ class BackblazeB2Service
public function downloadFile($fileID)
{
- $downloadToken = $this->getDownloadAuthToken();
-
return $this->sendRequest(
- sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)),
+ sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID)),
'GET',
null,
[
'http_headers' => [
- sprintf('Authorization: %s', $downloadToken)
+ sprintf('Authorization: %s', $this->authToken)
],
'response_body_is_json' => false
]
@@ -155,6 +165,35 @@ class BackblazeB2Service
throw new \Exception('No upload URL/authorization token returned from Backblaze B2.');
}
+ $exponentialBackoff = 1;
+ $numberOfRetries = 5; // this effectively gives us 31 seconds of retries (1+2+4+8+16)
+ $numberOfTimesTried = 0;
+
+ while ($numberOfTimesTried < $numberOfRetries)
+ {
+ try
+ {
+ return $this->uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken);
+ }
+ catch (BackblazeRetryException $ex)
+ {
+ sleep($exponentialBackoff);
+
+ // Get a new upload token
+ $this->uploadAuthToken = null;
+ $this->uploadUrl = null;
+
+ list($uploadUrl, $authorizationToken) = $this->getUploadUrl();
+
+ // Keep backing off
+ $exponentialBackoff *= $exponentialBackoff;
+ $numberOfTimesTried++;
+ }
+ }
+ }
+
+ private function uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken)
+ {
$fileSize = filesize($pathToFileToUpload);
$handle = fopen($pathToFileToUpload, 'r');
$fileContents = fread($handle, $fileSize);
@@ -168,7 +207,7 @@ class BackblazeB2Service
sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage))
];
- $result = $this->sendRequest(
+ $result = $this->sendRequestReal(
$uploadUrl,
'POST',
$fileContents,
@@ -200,15 +239,21 @@ class BackblazeB2Service
throw new \Exception(sprintf('The bucket \'%s\' was not found or your API key does not have access.', $bucketName));
}
- private function getUploadUrl()
+ private function getUploadUrl($alwaysGetNewToken = false)
{
- $result = $this->sendRequest(
- sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl),
- 'POST',
- ['bucketId' => $this->bucketId]
- );
+ if (is_null($this->uploadAuthToken) || $alwaysGetNewToken)
+ {
+ $result = $this->sendRequest(
+ sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl),
+ 'POST',
+ ['bucketId' => $this->bucketId]
+ );
- return [$result->uploadUrl, $result->authorizationToken];
+ $this->uploadAuthToken = $result->authorizationToken;
+ $this->uploadUrl = $result->uploadUrl;
+ }
+
+ return [$this->uploadUrl, $this->uploadAuthToken];
}
private function getBasicHttpClient($url, $method = 'GET', array $httpHeaders = [])
@@ -252,7 +297,15 @@ class BackblazeB2Service
}
catch (BackblazeRetryException $ex)
{
+ // Clear the upload token if requested
+ if (isset($postOptions['clear_upload_token_on_retry']) && $postOptions['clear_upload_token_on_retry'])
+ {
+ $this->uploadAuthToken = null;
+ $this->uploadUrl = null;
+ }
+
// Keep backing off
+ sleep($exponentialBackoff);
$exponentialBackoff *= $exponentialBackoff;
$numberOfTimesTried++;
}
@@ -318,7 +371,12 @@ class BackblazeB2Service
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
Log::info(sprintf('Received HTTP code %d', $httpCode));
- Log::debug($result);
+
+ // Only log a result if we have one and it's in JSON format (i.e. not a file download)
+ if (!is_null($result) && $result !== false && $postOptions['response_body_is_json'])
+ {
+ Log::debug($result);
+ }
// According to the Backblaze B2 Protocol, if we get a 500/503, we should retry the request
if ($httpCode == 500 || $httpCode == 503)
diff --git a/public/b2_test.php b/public/b2_test.php
new file mode 100644
index 0000000..a741604
--- /dev/null
+++ b/public/b2_test.php
@@ -0,0 +1,143 @@
+' . $uri . '
';
+
+ $server_output = curl_exec($session); // Let's do this!
+
+ if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
+ {
+ echo '' . $server_output . '
';
+ }
+ else
+ {
+ echo '' . (strlen($server_output) . ' bytes received') . '
'; // Tell me about the rabbits, George!
+ }
+
+ curl_close ($session); // Clean up
+
+ //$download_url = ""; // From b2_authorize_account call
+ $file_id = "4_z731245f41efc196b6dda0018_f116729ca6de74b38_d20190910_m132847_c002_v0001127_t0021"; // The ID of the file you want to download
+ $uri = $download_url . "/b2api/v2/b2_download_file_by_id?fileId=" . $file_id . '&Authorization=' . $auth_token;
+
+ $session = curl_init($uri);
+
+ curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
+ curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
+
+ echo '' . $uri . '
';
+
+ $server_output = curl_exec($session); // Let's do this!
+
+ if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
+ {
+ echo '' . $server_output . '
';
+ }
+ else
+ {
+ echo '' . (strlen($server_output) . ' bytes received') . '
'; // Tell me about the rabbits, George!
+ }
+
+ curl_close ($session); // Clean up
+}
+
+function b2_download_file_by_name($download_url, $auth_token)
+{
+ //$download_url = ""; // From b2_authorize_account call
+ $bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from
+ $file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download
+ $uri = $download_url . "/file/" . $bucket_name . "/" . $file_name;
+
+ $session = curl_init($uri);
+
+ curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET
+ curl_setopt($session, CURLOPT_SSL_VERIFYPEER, false);
+ curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
+
+ echo '' . $uri . '
';
+
+ $server_output = curl_exec($session); // Let's do this!
+
+ if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
+ {
+ echo '' . $server_output . '
';
+ }
+ else
+ {
+ echo '' . (strlen($server_output) . ' bytes received') . '
'; // Tell me about the rabbits, George!
+ }
+
+ curl_close ($session); // Clean up
+
+ // You will need to use the account authorization token if your bucket's type is allPrivate.
+
+ //$download_url = ""; // From b2_authorize_account call
+ $bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from
+ $file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download
+ //$auth_token = ""; // From b2_authorize_account call
+ $uri = $download_url . "/file/" . $bucket_name . "/" . $file_name . '?Authorization=' . $auth_token;
+
+ $session = curl_init($uri);
+
+ curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP POST
+ curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response
+
+ echo '' . $uri . '
';
+
+ $server_output = curl_exec($session); // Let's do this!
+
+ if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200)
+ {
+ echo '' . $server_output . '
';
+ }
+ else
+ {
+ echo '' . (strlen($server_output) . ' bytes received') . '
'; // Tell me about the rabbits, George!
+ }
+
+ curl_close ($session); // Clean up
+}
+
+?>
+b2_authorize_account
+
+
+b2_download_file_by_name
+downloadUrl, $authorize_account_result->authorizationToken); ?>
+
+b2_download_file_by_id
+downloadUrl, $authorize_account_result->authorizationToken); ?>