From 820287dd2cd9376923d9444a9f116fa6c46cabf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Preu=C3=9F?= Date: Thu, 22 Jul 2021 16:42:01 +0200 Subject: [PATCH] Improve caching, performance and add dry-run --- .gitignore | 1 + app/Bunny/Filesystem/CompareOptions.php | 14 +++ app/Bunny/Filesystem/EdgeFile.php | 31 ++++++- app/Bunny/Filesystem/EdgeStorage.php | 53 +++++++++-- app/Bunny/Filesystem/EdgeStorageCache.php | 68 ++++++++++++++ .../Exceptions/FileNotFoundException.php | 13 +++ .../Exceptions/FilesystemException.php | 22 +++++ app/Bunny/Filesystem/File.php | 4 +- app/Bunny/Filesystem/FileCompare.php | 89 ++++++++++++++----- app/Bunny/Filesystem/FileSerialized.php | 10 +++ app/Bunny/Filesystem/FileStreamable.php | 11 +++ app/Bunny/Filesystem/LocalFile.php | 32 ++++++- app/Commands/DeployCommand.php | 34 +++++-- 13 files changed, 342 insertions(+), 40 deletions(-) create mode 100644 app/Bunny/Filesystem/CompareOptions.php create mode 100644 app/Bunny/Filesystem/EdgeStorageCache.php create mode 100644 app/Bunny/Filesystem/Exceptions/FileNotFoundException.php create mode 100644 app/Bunny/Filesystem/Exceptions/FilesystemException.php create mode 100644 app/Bunny/Filesystem/FileSerialized.php create mode 100644 app/Bunny/Filesystem/FileStreamable.php diff --git a/.gitignore b/.gitignore index 99f9ef9..50fb19d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ .env *.bk /dist* +*.sha256 diff --git a/app/Bunny/Filesystem/CompareOptions.php b/app/Bunny/Filesystem/CompareOptions.php new file mode 100644 index 0000000..6abf001 --- /dev/null +++ b/app/Bunny/Filesystem/CompareOptions.php @@ -0,0 +1,14 @@ +file = $file; } + public static function fromFilename(string $path, string $sha256 = null): self + { + return new self((object)[ + 'Path' => Str::replaceLast($basename = basename($path), '', $path), + 'ObjectName' => $basename, + 'IsDirectory' => Str::endsWith($path, '/'), + 'Checksum' => $sha256, + ]); + } + public function getFilename($search = '', $replace = ''): string { return Str::replaceFirst($search, $replace, $this->file->Path . $this->file->ObjectName); @@ -21,11 +31,24 @@ class EdgeFile implements File public function isDirectory(): bool { - return $this->file->IsDirectory; + return $this->getChecksum() === null; } - public function getChecksum(): string + public function getChecksum(): ?string { - return $this->file->Checksum; + return $this->file->IsDirectory ? null : $this->file->Checksum; + } + + public function toArray(): array + { + return [ + 'sha256' => $this->getChecksum(), + 'filename' => $this->getFilename(), + ]; + } + + public static function fromArray(array $array): self + { + return self::fromFilename($array['filename'], $array['sha256']); } } diff --git a/app/Bunny/Filesystem/EdgeStorage.php b/app/Bunny/Filesystem/EdgeStorage.php index beda210..eb7adbe 100644 --- a/app/Bunny/Filesystem/EdgeStorage.php +++ b/app/Bunny/Filesystem/EdgeStorage.php @@ -2,26 +2,35 @@ namespace App\Bunny\Filesystem; +use App\Bunny\Filesystem\Exceptions\FileNotFoundException; +use App\Bunny\Filesystem\Exceptions\FilesystemException; use GuzzleHttp\Client; +use GuzzleHttp\Exception\ClientException; +use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\RequestOptions; -use Illuminate\Console\OutputStyle; use Illuminate\Support\Str; use Psr\Http\Message\ResponseInterface; class EdgeStorage { private Client $client; + private EdgeStorageCache $storageCache; public function __construct() { + $this->storageCache = new EdgeStorageCache($this); $this->client = new Client([ 'base_uri' => sprintf('https://%s', config('bunny.storage.hostname')), + 'http_errors' => false, + 'headers' => [ + 'User-Agent' => 'BunnyCLI/0.1', + ] ]); } - public function allFiles(string $path, callable $advance = null, &$results = array()) + public function allFiles(string $path, callable $advance = null, &$results = array()): array { $promise = $this->client->getAsync(self::normalizePath($path, true), [ RequestOptions::HEADERS => [ @@ -48,8 +57,7 @@ class EdgeStorage } }, function (RequestException $e) { - echo $e->getMessage() . "\n"; - echo $e->getRequest()->getMethod(); + throw FilesystemException::fromPrevious($e); } ); @@ -58,14 +66,42 @@ class EdgeStorage return $results; } - public function put(LocalFile $file, string $local, string $edge): PromiseInterface + /** + * @throws FilesystemException + */ + public function get(EdgeFile $file): string + { + try { + $response = $this->client->get(self::normalizePath($file->getFilename(), $file->isDirectory()), [ + RequestOptions::HEADERS => [ + 'AccessKey' => config('bunny.storage.password'), + ], + ]); + } catch (ClientException $exception) { + throw FilesystemException::fromResponse($exception->getResponse()); + } catch (GuzzleException $exception) { + throw FilesystemException::fromPrevious($exception); + } + + if ($response->getStatusCode() === 404) { + throw FileNotFoundException::fromFile($file); + } + + if ($response->getStatusCode() !== 200) { + throw FilesystemException::fromResponse($response); + } + + return $response->getBody()->getContents(); + } + + public function put(LocalFile $file, string $local = '', string $edge = ''): PromiseInterface { return $this->client->putAsync(self::normalizePath($file->getFilename($local, $edge), $file->isDirectory()), [ RequestOptions::HEADERS => [ 'AccessKey' => config('bunny.storage.password'), 'Checksum' => $file->getChecksum(), ], - RequestOptions::BODY => fopen($file->getFilename(), 'r'), + RequestOptions::BODY => $file->getResource(), ]); } @@ -83,6 +119,11 @@ class EdgeStorage return $this->client; } + public function getStorageCache(): EdgeStorageCache + { + return $this->storageCache; + } + private static function normalizePath(string $filename, bool $isDirectory): string { if (!Str::startsWith($filename, ['/'])) { diff --git a/app/Bunny/Filesystem/EdgeStorageCache.php b/app/Bunny/Filesystem/EdgeStorageCache.php new file mode 100644 index 0000000..ddf42e2 --- /dev/null +++ b/app/Bunny/Filesystem/EdgeStorageCache.php @@ -0,0 +1,68 @@ +edgeStorage = $edgeStorage; + } + + /** + * @throws FilesystemException + */ + public function parse(string $path): array + { + $contents = $this->edgeStorage->get(EdgeFile::fromFilename( + sprintf('%s/%s', $path, $this->filename), + File::EMPTY_SHA256, + )); + + file_put_contents(base_path(sprintf('%s.bk', basename($this->filename))), $contents); + + return $this->extract($contents); + } + + public function save(string $local, string $edge, array $files): bool + { + $filename = sprintf('%s/%s', $edge, $this->filename); + $contents = $this->hydrate($files, $local, $edge); + $checksum = strtoupper(hash('sha256', $contents)); + + file_put_contents(base_path(sprintf('%s', basename($this->filename))), $contents); + + $promise = $this->edgeStorage->put(new LocalFile($filename, $checksum, $contents)); + + /** @var ResponseInterface $response */ + $response = $promise->wait(); + + return $response->getStatusCode() === 200; + } + + private function extract(string $contents): array + { + if (!$array = json_decode($contents, true)) { + throw new FileNotFoundException('Cannot parse cache file.'); + } + + return array_map(fn(array $x) => EdgeFile::fromArray($x), $array); + } + + private function hydrate(array $files, string $search = '', string $replace = ''): string + { + return json_encode(array_map(fn(LocalFile $x) => $x->toArray($search, $replace), $files), JSON_PRETTY_PRINT); + } + + public function setFilename(string $filename) + { + $this->filename = $filename; + } +} diff --git a/app/Bunny/Filesystem/Exceptions/FileNotFoundException.php b/app/Bunny/Filesystem/Exceptions/FileNotFoundException.php new file mode 100644 index 0000000..a1d6ceb --- /dev/null +++ b/app/Bunny/Filesystem/Exceptions/FileNotFoundException.php @@ -0,0 +1,13 @@ +getFilename())); + } +} diff --git a/app/Bunny/Filesystem/Exceptions/FilesystemException.php b/app/Bunny/Filesystem/Exceptions/FilesystemException.php new file mode 100644 index 0000000..9ff2ff4 --- /dev/null +++ b/app/Bunny/Filesystem/Exceptions/FilesystemException.php @@ -0,0 +1,22 @@ +getStatusCode() + )); + } + + public static function fromPrevious(Exception $exception): self + { + return new self($exception->getMessage(), 0, $exception); + } +} diff --git a/app/Bunny/Filesystem/File.php b/app/Bunny/Filesystem/File.php index 61f5aab..9088e0e 100644 --- a/app/Bunny/Filesystem/File.php +++ b/app/Bunny/Filesystem/File.php @@ -4,9 +4,11 @@ namespace App\Bunny\Filesystem; interface File { + public const EMPTY_SHA256 = '0000000000000000000000000000000000000000000000000000000000000000'; + public function getFilename($search = '', $replace = ''): string; - public function getChecksum(): string; + public function getChecksum(): ?string; public function isDirectory(): bool; } diff --git a/app/Bunny/Filesystem/FileCompare.php b/app/Bunny/Filesystem/FileCompare.php index 43a499d..00576c9 100644 --- a/app/Bunny/Filesystem/FileCompare.php +++ b/app/Bunny/Filesystem/FileCompare.php @@ -3,6 +3,8 @@ namespace App\Bunny\Filesystem; use App\Bunny\Client; +use App\Bunny\Filesystem\Exceptions\FileNotFoundException; +use Exception; use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Pool; use GuzzleHttp\Psr7\Response; @@ -25,18 +27,17 @@ class FileCompare $this->command = $command; } - public function compare(string $local, string $edge, float $start): void + /** + * @throws Exceptions\FilesystemException + */ + public function compare(string $local, string $edge, array $options): void { $this->command->info('- Hashing files...'); $localFilesAndDirectories = $this->localStorage->allFiles($local); $localFiles = array_filter($localFilesAndDirectories, fn(LocalFile $x) => !$x->isDirectory()); $this->command->info(sprintf('✔ Finished hashing %s files', count($localFiles))); - $this->command->info('- CDN fetching files and directories (progress is approximately)...'); $expectedMax = count($localFilesAndDirectories) - count($localFiles); - $bar = $this->command->getOutput()->createProgressBar($expectedMax); - $edgeFiles = $this->edgeStorage->allFiles($edge, fn() => $bar->advance()); - $bar->finish(); - $this->command->newLine(); + $edgeFiles = $this->getEdgeFiles($options, $edge, $expectedMax); $this->command->info(sprintf('✔ Finished fetching %s files and directories', count($edgeFiles))); $this->command->info('- CDN diffing files and directories...'); @@ -93,33 +94,45 @@ class FileCompare }, ]); - // Initiate the transfers and create a promise - $promise = $pool->promise(); + if (!$options[CompareOptions::DRY_RUN]) { + // Initiate the transfers and create a promise + $promise = $pool->promise(); - $bar->start(); - $promise->wait(); // Force the pool of requests to complete. - $bar->finish(); + $bar->start(); + $promise->wait(); // Force the pool of requests to complete. + $bar->finish(); - $this->command->newLine(); + $this->command->newLine(); + } $this->command->info(sprintf('✔ Finished synchronizing %s', $type)); } } + if (!$options[CompareOptions::NO_SHA256_GENERATION]) { + $this->command->info('- Generating cache for current deployment...'); + + if (!$this->edgeStorage->getStorageCache()->save($local, $edge, $localFilesAndDirectories)) { + $this->command->info('✔ Cache published successfully.'); + } else { + $this->command->error('✘ Error publishing cache.'); + } + } + $pullZoneId = config('bunny.pull_zone.id'); + $this->command->info('- Waiting for deploy to go live...'); - $result = $this->apiClient->purgeCache($pullZoneId = config('bunny.pull_zone.id')); - - if (!$result->success()) { - $this->command->info('✔ Deploy is live (without flush)!'); - return; + if (!$options[CompareOptions::DRY_RUN]) { + $flushResult = $this->apiClient->purgeCache($pullZoneId); } $result = $this->apiClient->getPullZone($pullZoneId); - $timeElapsedSecs = microtime(true) - $start; - - $this->command->info(sprintf('✔ Deployment is live! (%ss)', number_format($timeElapsedSecs, 2))); + $timeElapsedSecs = microtime(true) - $options[CompareOptions::START]; + $message = !isset($flushResult) || !$result->success() + ? '✔ Deployment is live (without flush)! (%ss)' + : '✔ Deployment is live! (%ss)'; + $this->command->info(sprintf($message, number_format($timeElapsedSecs, 2))); $this->command->newLine(); foreach ($result->getData()->Hostnames as $hostname) { @@ -149,4 +162,40 @@ class FileCompare return $reason->getRequest()->getMethod() === 'DELETE' && in_array($reason->getResponse()->getStatusCode(), [404, 400, 500], true); } + + /** + * @throws Exceptions\FilesystemException + */ + private function getEdgeFiles(array $options, string $edge, int $expectedMax): array + { + $this->edgeStorage->getStorageCache()->setFilename($options[CompareOptions::SHA256_NAME]); + + if ($options[CompareOptions::NO_SHA256_CACHE]) { + return $this->getAllFilesRecursive($expectedMax, $edge); + } + + try { + $this->command->info('- CDN fetching files and directories from cache...'); + return $this->edgeStorage->getStorageCache()->parse($edge); + } catch (FileNotFoundException $exception) { + $this->command->warn(sprintf( + '⚠ Cannot fetch %s from storage due "%s". Using recursive fallback...', + $options[CompareOptions::SHA256_NAME], + $exception->getMessage() + )); + return $this->getAllFilesRecursive($expectedMax, $edge); + } catch (Exception $e) { + throw $e; + } + } + + private function getAllFilesRecursive(int $expectedMax, string $edge): array + { + $this->command->info('- CDN fetching files and directories (progress is approximately)...'); + $bar = $this->command->getOutput()->createProgressBar($expectedMax); + $result = $this->edgeStorage->allFiles($edge, fn() => $bar->advance()); + $bar->finish(); + $this->command->newLine(); + return $result; + } } diff --git a/app/Bunny/Filesystem/FileSerialized.php b/app/Bunny/Filesystem/FileSerialized.php new file mode 100644 index 0000000..88630ca --- /dev/null +++ b/app/Bunny/Filesystem/FileSerialized.php @@ -0,0 +1,10 @@ +filename = $filename; $this->checksum = $checksum; + $this->contents = $contents; } public function getFilename($search = '', $replace = ''): string @@ -20,13 +22,35 @@ class LocalFile implements File return Str::replaceFirst($search, $replace, $this->filename); } - public function getChecksum(): string + public function getChecksum(): ?string { return $this->checksum; } public function isDirectory(): bool { - return $this->checksum == null; + return $this->getChecksum() === null; + } + + public function getResource() + { + if ($this->contents) { + return $this->contents; + } + + return fopen($this->getFilename(), 'r'); + } + + public function toArray(string $search = '', string $replace = ''): array + { + return [ + 'sha256' => $this->getChecksum(), + 'filename' => $this->getFilename($search, $replace), + ]; + } + + public static function fromArray(array $array): self + { + return new self($array['filename'], $array['sha256']); } } diff --git a/app/Commands/DeployCommand.php b/app/Commands/DeployCommand.php index 03feccb..9e1dd8a 100644 --- a/app/Commands/DeployCommand.php +++ b/app/Commands/DeployCommand.php @@ -2,7 +2,9 @@ namespace App\Commands; +use App\Bunny\Filesystem\CompareOptions; use App\Bunny\Filesystem\EdgeStorage; +use App\Bunny\Filesystem\Exceptions\FilesystemException; use App\Bunny\Filesystem\FileCompare; use App\Bunny\Filesystem\LocalStorage; use Illuminate\Console\Scheduling\Schedule; @@ -16,7 +18,11 @@ class DeployCommand extends Command * @var string */ protected $signature = 'deploy - {--dir=dist : Root directory to upload}'; + {--dir=dist : Root directory to upload} + {--no-sha256-cache : Skips .well-known/bunny.sha256 and queries the storage endpoints recursively instead} + {--no-sha256-generation : Skips .well-known/bunny.sha256 generation} + {--sha256-name=.well-known/bunny.sha256 : Change filename of .well-known/bunny.sha256} + {--dry-run : Outputs the operations but will not execute anything}'; /** * The description of the command. @@ -34,9 +40,11 @@ class DeployCommand extends Command { $start = microtime(true); - $edgeStorage = new EdgeStorage(); - $localStorage = new LocalStorage(); - $fileCompare = new FileCompare($localStorage, $edgeStorage, $this); + $fileCompare = new FileCompare( + app(LocalStorage::class), + app(EdgeStorage::class), + $this + ); $localPath = realpath($path = $this->option('dir') ?? 'dist'); @@ -47,7 +55,23 @@ class DeployCommand extends Command $edgePath = sprintf('/%s', config('bunny.storage.username')); - $fileCompare->compare($localPath, $edgePath, $start); + if ($this->option('dry-run')) { + $this->warn('⚠ Dry run is activated. The operations are displayed but not executed.'); + } + + try { + $fileCompare->compare($localPath, $edgePath, [ + CompareOptions::START => $start, + CompareOptions::NO_SHA256_CACHE => $this->option('no-sha256-cache'), + CompareOptions::NO_SHA256_GENERATION => $this->option('no-sha256-generation'), + CompareOptions::SHA256_NAME => $this->option('sha256-name'), + CompareOptions::DRY_RUN => $this->option('dry-run'), + ]); + } catch (FilesystemException $exception) { + $this->error($exception->getMessage()); + + return 2; + } return 0; }