From a6825bcef90bdaa7f05759047945540af6cacf20 Mon Sep 17 00:00:00 2001 From: Andy Heathershaw Date: Sat, 14 Sep 2019 10:04:09 +0100 Subject: [PATCH] Backblaze #135 - implemented the re-use of the upload token/URL. Fetching file contents now works by using the b2_download_file_by_id method with an auth header. --- app/AlbumSources/BackblazeB2Source.php | 20 ++-- app/Services/BackblazeB2Service.php | 84 ++++++++++++--- public/b2_test.php | 143 +++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 public/b2_test.php diff --git a/app/AlbumSources/BackblazeB2Source.php b/app/AlbumSources/BackblazeB2Source.php index c9a4c9e..5223d32 100644 --- a/app/AlbumSources/BackblazeB2Source.php +++ b/app/AlbumSources/BackblazeB2Source.php @@ -74,14 +74,22 @@ class BackblazeB2Source extends AlbumSourceBase implements IAlbumSource */ public function fetchPhotoContent(Photo $photo, $thumbnail = null) { - // Use the same URLs that the public would use to fetch the file - $urlToPhoto = $this->getUrlToPhoto($photo, $thumbnail); + $pathOnStorage = $this->getPathToPhoto($photo, $thumbnail); - $ch = curl_init($urlToPhoto); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - $fileContent = curl_exec($ch); + // First we need the file ID - return EntityBody::fromString($fileContent); + /** @var BackblazeB2FileIdCache $b2Cache */ + $b2Cache = BackblazeB2FileIdCache::where('storage_path', $pathOnStorage)->first(); + if (is_null($b2Cache)) + { + // TODO: lookup the file on B2 to get the file ID + Log::warning(sprintf('B2 file ID not found in cache: %s', $pathOnStorage)); + return EntityBody::fromString(''); + } + + return EntityBody::fromString( + $this->getClient()->downloadFile($b2Cache->b2_file_id) + ); } /** diff --git a/app/Services/BackblazeB2Service.php b/app/Services/BackblazeB2Service.php index 57149ee..ceab613 100644 --- a/app/Services/BackblazeB2Service.php +++ b/app/Services/BackblazeB2Service.php @@ -55,6 +55,18 @@ class BackblazeB2Service */ private $config; + /** + * Current file upload token. + * @var string + */ + private $uploadAuthToken; + + /** + * Current upload URL. + * @var string + */ + private $uploadUrl; + public function __construct() { $this->config = config('services.backblaze_b2'); @@ -92,15 +104,13 @@ class BackblazeB2Service public function downloadFile($fileID) { - $downloadToken = $this->getDownloadAuthToken(); - return $this->sendRequest( - sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID), urlencode($downloadToken)), + sprintf('%s/b2api/v2/b2_download_file_by_id?fileId=%s', $this->accountApiUrl, urlencode($fileID)), 'GET', null, [ 'http_headers' => [ - sprintf('Authorization: %s', $downloadToken) + sprintf('Authorization: %s', $this->authToken) ], 'response_body_is_json' => false ] @@ -155,6 +165,35 @@ class BackblazeB2Service throw new \Exception('No upload URL/authorization token returned from Backblaze B2.'); } + $exponentialBackoff = 1; + $numberOfRetries = 5; // this effectively gives us 31 seconds of retries (1+2+4+8+16) + $numberOfTimesTried = 0; + + while ($numberOfTimesTried < $numberOfRetries) + { + try + { + return $this->uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken); + } + catch (BackblazeRetryException $ex) + { + sleep($exponentialBackoff); + + // Get a new upload token + $this->uploadAuthToken = null; + $this->uploadUrl = null; + + list($uploadUrl, $authorizationToken) = $this->getUploadUrl(); + + // Keep backing off + $exponentialBackoff *= $exponentialBackoff; + $numberOfTimesTried++; + } + } + } + + private function uploadFileReal($pathToFileToUpload, $pathToStorage, $uploadUrl, $authorizationToken) + { $fileSize = filesize($pathToFileToUpload); $handle = fopen($pathToFileToUpload, 'r'); $fileContents = fread($handle, $fileSize); @@ -168,7 +207,7 @@ class BackblazeB2Service sprintf('X-Bz-File-Name: %s', urlencode($pathToStorage)) ]; - $result = $this->sendRequest( + $result = $this->sendRequestReal( $uploadUrl, 'POST', $fileContents, @@ -200,15 +239,21 @@ class BackblazeB2Service throw new \Exception(sprintf('The bucket \'%s\' was not found or your API key does not have access.', $bucketName)); } - private function getUploadUrl() + private function getUploadUrl($alwaysGetNewToken = false) { - $result = $this->sendRequest( - sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl), - 'POST', - ['bucketId' => $this->bucketId] - ); + if (is_null($this->uploadAuthToken) || $alwaysGetNewToken) + { + $result = $this->sendRequest( + sprintf('%s/b2api/v2/b2_get_upload_url', $this->accountApiUrl), + 'POST', + ['bucketId' => $this->bucketId] + ); - return [$result->uploadUrl, $result->authorizationToken]; + $this->uploadAuthToken = $result->authorizationToken; + $this->uploadUrl = $result->uploadUrl; + } + + return [$this->uploadUrl, $this->uploadAuthToken]; } private function getBasicHttpClient($url, $method = 'GET', array $httpHeaders = []) @@ -252,7 +297,15 @@ class BackblazeB2Service } catch (BackblazeRetryException $ex) { + // Clear the upload token if requested + if (isset($postOptions['clear_upload_token_on_retry']) && $postOptions['clear_upload_token_on_retry']) + { + $this->uploadAuthToken = null; + $this->uploadUrl = null; + } + // Keep backing off + sleep($exponentialBackoff); $exponentialBackoff *= $exponentialBackoff; $numberOfTimesTried++; } @@ -318,7 +371,12 @@ class BackblazeB2Service $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); Log::info(sprintf('Received HTTP code %d', $httpCode)); - Log::debug($result); + + // Only log a result if we have one and it's in JSON format (i.e. not a file download) + if (!is_null($result) && $result !== false && $postOptions['response_body_is_json']) + { + Log::debug($result); + } // According to the Backblaze B2 Protocol, if we get a 500/503, we should retry the request if ($httpCode == 500 || $httpCode == 503) diff --git a/public/b2_test.php b/public/b2_test.php new file mode 100644 index 0000000..a741604 --- /dev/null +++ b/public/b2_test.php @@ -0,0 +1,143 @@ +' . $uri . '

'; + + $server_output = curl_exec($session); // Let's do this! + + if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200) + { + echo '

' . $server_output . '

'; + } + else + { + echo '

' . (strlen($server_output) . ' bytes received') . '

'; // Tell me about the rabbits, George! + } + + curl_close ($session); // Clean up + + //$download_url = ""; // From b2_authorize_account call + $file_id = "4_z731245f41efc196b6dda0018_f116729ca6de74b38_d20190910_m132847_c002_v0001127_t0021"; // The ID of the file you want to download + $uri = $download_url . "/b2api/v2/b2_download_file_by_id?fileId=" . $file_id . '&Authorization=' . $auth_token; + + $session = curl_init($uri); + + curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET + curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response + + echo '

' . $uri . '

'; + + $server_output = curl_exec($session); // Let's do this! + + if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200) + { + echo '

' . $server_output . '

'; + } + else + { + echo '

' . (strlen($server_output) . ' bytes received') . '

'; // Tell me about the rabbits, George! + } + + curl_close ($session); // Clean up +} + +function b2_download_file_by_name($download_url, $auth_token) +{ + //$download_url = ""; // From b2_authorize_account call + $bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from + $file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download + $uri = $download_url . "/file/" . $bucket_name . "/" . $file_name; + + $session = curl_init($uri); + + curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP GET + curl_setopt($session, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response + + echo '

' . $uri . '

'; + + $server_output = curl_exec($session); // Let's do this! + + if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200) + { + echo '

' . $server_output . '

'; + } + else + { + echo '

' . (strlen($server_output) . ' bytes received') . '

'; // Tell me about the rabbits, George! + } + + curl_close ($session); // Clean up + + // You will need to use the account authorization token if your bucket's type is allPrivate. + + //$download_url = ""; // From b2_authorize_account call + $bucket_name = "andysh-bt-test"; // The NAME of the bucket you want to download from + $file_name = "B2-Test-Album/preview/7tgoy55do1vjv180ytlp.jpeg"; // The name of the file you want to download + //$auth_token = ""; // From b2_authorize_account call + $uri = $download_url . "/file/" . $bucket_name . "/" . $file_name . '?Authorization=' . $auth_token; + + $session = curl_init($uri); + + curl_setopt($session, CURLOPT_HTTPGET, true); // HTTP POST + curl_setopt($session, CURLOPT_RETURNTRANSFER, true); // Receive server response + + echo '

' . $uri . '

'; + + $server_output = curl_exec($session); // Let's do this! + + if (curl_getinfo($session, CURLINFO_HTTP_CODE) != 200) + { + echo '

' . $server_output . '

'; + } + else + { + echo '

' . (strlen($server_output) . ' bytes received') . '

'; // Tell me about the rabbits, George! + } + + curl_close ($session); // Clean up +} + +?> +

b2_authorize_account

+ + +

b2_download_file_by_name

+downloadUrl, $authorize_account_result->authorizationToken); ?> + +

b2_download_file_by_id

+downloadUrl, $authorize_account_result->authorizationToken); ?>