Improve caching, performance and add dry-run

.gitignore

@@ -8,3 +8,4 @@
 .env
 *.bk
 /dist*
+*.sha256

app/Bunny/Filesystem/CompareOptions.php

@@ -0,0 +1,14 @@
+<?php
+
+
+namespace App\Bunny\Filesystem;
+
+
+class CompareOptions
+{
+    const START = 'start';
+    const NO_SHA256_CACHE = 'no_sha256_cache';
+    const NO_SHA256_GENERATION = 'no_sha256_generation';
+    const SHA256_NAME = 'sha256_name';
+    const DRY_RUN = 'dry-run';
+}

app/Bunny/Filesystem/EdgeFile.php

@@ -5,7 +5,7 @@ namespace App\Bunny\Filesystem;
 use Illuminate\Support\Str;
 use stdClass;
 
-class EdgeFile implements File
+class EdgeFile implements File, FileSerialized
 {
     private stdClass $file;
 
@@ -14,6 +14,16 @@ class EdgeFile implements File
         $this->file = $file;
     }
 
+    public static function fromFilename(string $path, string $sha256 = null): self
+    {
+        return new self((object)[
+            'Path' => Str::replaceLast($basename = basename($path), '', $path),
+            'ObjectName' => $basename,
+            'IsDirectory' => Str::endsWith($path, '/'),
+            'Checksum' => $sha256,
+        ]);
+    }
+
     public function getFilename($search = '', $replace = ''): string
     {
         return Str::replaceFirst($search, $replace, $this->file->Path . $this->file->ObjectName);
@@ -21,11 +31,24 @@ class EdgeFile implements File
 
     public function isDirectory(): bool
     {
-        return $this->file->IsDirectory;
+        return $this->getChecksum() === null;
     }
 
-    public function getChecksum(): string
+    public function getChecksum(): ?string
     {
-        return $this->file->Checksum;
+        return $this->file->IsDirectory ? null : $this->file->Checksum;
+    }
+
+    public function toArray(): array
+    {
+        return [
+            'sha256' => $this->getChecksum(),
+            'filename' => $this->getFilename(),
+        ];
+    }
+
+    public static function fromArray(array $array): self
+    {
+        return self::fromFilename($array['filename'], $array['sha256']);
     }
 }

app/Bunny/Filesystem/EdgeStorage.php

@@ -2,26 +2,35 @@
 
 namespace App\Bunny\Filesystem;
 
+use App\Bunny\Filesystem\Exceptions\FileNotFoundException;
+use App\Bunny\Filesystem\Exceptions\FilesystemException;
 use GuzzleHttp\Client;
+use GuzzleHttp\Exception\ClientException;
+use GuzzleHttp\Exception\GuzzleException;
 use GuzzleHttp\Exception\RequestException;
 use GuzzleHttp\Promise\PromiseInterface;
 use GuzzleHttp\RequestOptions;
-use Illuminate\Console\OutputStyle;
 use Illuminate\Support\Str;
 use Psr\Http\Message\ResponseInterface;
 
 class EdgeStorage
 {
     private Client $client;
+    private EdgeStorageCache $storageCache;
 
     public function __construct()
     {
+        $this->storageCache = new EdgeStorageCache($this);
         $this->client = new Client([
             'base_uri' => sprintf('https://%s', config('bunny.storage.hostname')),
+            'http_errors' => false,
+            'headers' => [
+                'User-Agent' => 'BunnyCLI/0.1',
+            ]
         ]);
     }
 
-    public function allFiles(string $path, callable $advance = null, &$results = array())
+    public function allFiles(string $path, callable $advance = null, &$results = array()): array
     {
         $promise = $this->client->getAsync(self::normalizePath($path, true), [
             RequestOptions::HEADERS => [
@@ -48,8 +57,7 @@ class EdgeStorage
                 }
             },
             function (RequestException $e) {
-                echo $e->getMessage() . "\n";
-                echo $e->getRequest()->getMethod();
+                throw FilesystemException::fromPrevious($e);
             }
         );
 
@@ -58,14 +66,42 @@ class EdgeStorage
         return $results;
     }
 
-    public function put(LocalFile $file, string $local, string $edge): PromiseInterface
+    /**
+     * @throws FilesystemException
+     */
+    public function get(EdgeFile $file): string
+    {
+        try {
+            $response = $this->client->get(self::normalizePath($file->getFilename(), $file->isDirectory()), [
+                RequestOptions::HEADERS => [
+                    'AccessKey' => config('bunny.storage.password'),
+                ],
+            ]);
+        } catch (ClientException $exception) {
+            throw FilesystemException::fromResponse($exception->getResponse());
+        } catch (GuzzleException $exception) {
+            throw FilesystemException::fromPrevious($exception);
+        }
+
+        if ($response->getStatusCode() === 404) {
+            throw FileNotFoundException::fromFile($file);
+        }
+
+        if ($response->getStatusCode() !== 200) {
+            throw FilesystemException::fromResponse($response);
+        }
+
+        return $response->getBody()->getContents();
+    }
+
+    public function put(LocalFile $file, string $local = '', string $edge = ''): PromiseInterface
     {
         return $this->client->putAsync(self::normalizePath($file->getFilename($local, $edge), $file->isDirectory()), [
             RequestOptions::HEADERS => [
                 'AccessKey' => config('bunny.storage.password'),
                 'Checksum' => $file->getChecksum(),
             ],
-            RequestOptions::BODY => fopen($file->getFilename(), 'r'),
+            RequestOptions::BODY => $file->getResource(),
         ]);
     }
 
@@ -83,6 +119,11 @@ class EdgeStorage
         return $this->client;
     }
 
+    public function getStorageCache(): EdgeStorageCache
+    {
+        return $this->storageCache;
+    }
+
     private static function normalizePath(string $filename, bool $isDirectory): string
     {
         if (!Str::startsWith($filename, ['/'])) {

app/Bunny/Filesystem/EdgeStorageCache.php

@@ -0,0 +1,68 @@
+<?php
+
+namespace App\Bunny\Filesystem;
+
+use App\Bunny\Filesystem\Exceptions\FileNotFoundException;
+use App\Bunny\Filesystem\Exceptions\FilesystemException;
+use Psr\Http\Message\ResponseInterface;
+
+class EdgeStorageCache
+{
+    private EdgeStorage $edgeStorage;
+    private string $filename = '.well-known/bunny.sha256';
+
+    public function __construct(EdgeStorage $edgeStorage)
+    {
+        $this->edgeStorage = $edgeStorage;
+    }
+
+    /**
+     * @throws FilesystemException
+     */
+    public function parse(string $path): array
+    {
+        $contents = $this->edgeStorage->get(EdgeFile::fromFilename(
+            sprintf('%s/%s', $path, $this->filename),
+            File::EMPTY_SHA256,
+        ));
+
+        file_put_contents(base_path(sprintf('%s.bk', basename($this->filename))), $contents);
+
+        return $this->extract($contents);
+    }
+
+    public function save(string $local, string $edge, array $files): bool
+    {
+        $filename = sprintf('%s/%s', $edge, $this->filename);
+        $contents = $this->hydrate($files, $local, $edge);
+        $checksum = strtoupper(hash('sha256', $contents));
+
+        file_put_contents(base_path(sprintf('%s', basename($this->filename))), $contents);
+
+        $promise = $this->edgeStorage->put(new LocalFile($filename, $checksum, $contents));
+
+        /** @var ResponseInterface $response */
+        $response = $promise->wait();
+
+        return $response->getStatusCode() === 200;
+    }
+
+    private function extract(string $contents): array
+    {
+        if (!$array = json_decode($contents, true)) {
+            throw new FileNotFoundException('Cannot parse cache file.');
+        }
+
+        return array_map(fn(array $x) => EdgeFile::fromArray($x), $array);
+    }
+
+    private function hydrate(array $files, string $search = '', string $replace = ''): string
+    {
+        return json_encode(array_map(fn(LocalFile $x) => $x->toArray($search, $replace), $files), JSON_PRETTY_PRINT);
+    }
+
+    public function setFilename(string $filename)
+    {
+        $this->filename = $filename;
+    }
+}

app/Bunny/Filesystem/Exceptions/FileNotFoundException.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Bunny\Filesystem\Exceptions;
+
+use App\Bunny\Filesystem\File;
+
+class FileNotFoundException extends FilesystemException
+{
+    public static function fromFile(File $file): self
+    {
+        return new self(sprintf('The file %s was not found.', $file->getFilename()));
+    }
+}

app/Bunny/Filesystem/Exceptions/FilesystemException.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace App\Bunny\Filesystem\Exceptions;
+
+use Exception;
+use Psr\Http\Message\ResponseInterface;
+
+class FilesystemException extends Exception
+{
+    public static function fromResponse(ResponseInterface $response): self
+    {
+        return new self(sprintf(
+            'The storage api returns %s which is an invalid status code.',
+            $response->getStatusCode()
+        ));
+    }
+
+    public static function fromPrevious(Exception $exception): self
+    {
+        return new self($exception->getMessage(), 0, $exception);
+    }
+}

app/Bunny/Filesystem/File.php

@@ -4,9 +4,11 @@ namespace App\Bunny\Filesystem;
 
 interface File
 {
+    public const EMPTY_SHA256 = '0000000000000000000000000000000000000000000000000000000000000000';
+
     public function getFilename($search = '', $replace = ''): string;
 
-    public function getChecksum(): string;
+    public function getChecksum(): ?string;
 
     public function isDirectory(): bool;
 }

app/Bunny/Filesystem/FileCompare.php

@@ -3,6 +3,8 @@
 namespace App\Bunny\Filesystem;
 
 use App\Bunny\Client;
+use App\Bunny\Filesystem\Exceptions\FileNotFoundException;
+use Exception;
 use GuzzleHttp\Exception\RequestException;
 use GuzzleHttp\Pool;
 use GuzzleHttp\Psr7\Response;
@@ -25,18 +27,17 @@ class FileCompare
         $this->command = $command;
     }
 
-    public function compare(string $local, string $edge, float $start): void
+    /**
+     * @throws Exceptions\FilesystemException
+     */
+    public function compare(string $local, string $edge, array $options): void
     {
         $this->command->info('- Hashing files...');
         $localFilesAndDirectories = $this->localStorage->allFiles($local);
         $localFiles = array_filter($localFilesAndDirectories, fn(LocalFile $x) => !$x->isDirectory());
         $this->command->info(sprintf('✔ Finished hashing %s files', count($localFiles)));
-        $this->command->info('- CDN fetching files and directories (progress is approximately)...');
         $expectedMax = count($localFilesAndDirectories) - count($localFiles);
-        $bar = $this->command->getOutput()->createProgressBar($expectedMax);
-        $edgeFiles = $this->edgeStorage->allFiles($edge, fn() => $bar->advance());
-        $bar->finish();
-        $this->command->newLine();
+        $edgeFiles = $this->getEdgeFiles($options, $edge, $expectedMax);
         $this->command->info(sprintf('✔ Finished fetching %s files and directories', count($edgeFiles)));
         $this->command->info('- CDN diffing files and directories...');
 
@@ -93,6 +94,7 @@ class FileCompare
                     },
                 ]);
 
+                if (!$options[CompareOptions::DRY_RUN]) {
                     // Initiate the transfers and create a promise
                     $promise = $pool->promise();
 
@@ -101,25 +103,36 @@ class FileCompare
                     $bar->finish();
 
                     $this->command->newLine();
+                }
 
                 $this->command->info(sprintf('✔ Finished synchronizing %s', $type));
             }
         }
 
+        if (!$options[CompareOptions::NO_SHA256_GENERATION]) {
+            $this->command->info('- Generating cache for current deployment...');
+
+            if (!$this->edgeStorage->getStorageCache()->save($local, $edge, $localFilesAndDirectories)) {
+                $this->command->info('✔ Cache published successfully.');
+            } else {
+                $this->command->error('✘ Error publishing cache.');
+            }
+        }
+        $pullZoneId = config('bunny.pull_zone.id');
+
         $this->command->info('- Waiting for deploy to go live...');
 
-        $result = $this->apiClient->purgeCache($pullZoneId = config('bunny.pull_zone.id'));
-
-        if (!$result->success()) {
-            $this->command->info('✔ Deploy is live (without flush)!');
-            return;
+        if (!$options[CompareOptions::DRY_RUN]) {
+            $flushResult = $this->apiClient->purgeCache($pullZoneId);
         }
 
         $result = $this->apiClient->getPullZone($pullZoneId);
 
-        $timeElapsedSecs = microtime(true) - $start;
-
-        $this->command->info(sprintf('✔ Deployment is live! (%ss)', number_format($timeElapsedSecs, 2)));
+        $timeElapsedSecs = microtime(true) - $options[CompareOptions::START];
+        $message = !isset($flushResult) || !$result->success()
+            ? '✔ Deployment is live (without flush)! (%ss)'
+            : '✔ Deployment is live! (%ss)';
+        $this->command->info(sprintf($message, number_format($timeElapsedSecs, 2)));
         $this->command->newLine();
 
         foreach ($result->getData()->Hostnames as $hostname) {
@@ -149,4 +162,40 @@ class FileCompare
         return $reason->getRequest()->getMethod() === 'DELETE'
             && in_array($reason->getResponse()->getStatusCode(), [404, 400, 500], true);
     }
+
+    /**
+     * @throws Exceptions\FilesystemException
+     */
+    private function getEdgeFiles(array $options, string $edge, int $expectedMax): array
+    {
+        $this->edgeStorage->getStorageCache()->setFilename($options[CompareOptions::SHA256_NAME]);
+
+        if ($options[CompareOptions::NO_SHA256_CACHE]) {
+            return $this->getAllFilesRecursive($expectedMax, $edge);
+        }
+
+        try {
+            $this->command->info('- CDN fetching files and directories from cache...');
+            return $this->edgeStorage->getStorageCache()->parse($edge);
+        } catch (FileNotFoundException $exception) {
+            $this->command->warn(sprintf(
+                '⚠ Cannot fetch %s from storage due "%s". Using recursive fallback...',
+                $options[CompareOptions::SHA256_NAME],
+                $exception->getMessage()
+            ));
+            return $this->getAllFilesRecursive($expectedMax, $edge);
+        } catch (Exception $e) {
+            throw $e;
+        }
+    }
+
+    private function getAllFilesRecursive(int $expectedMax, string $edge): array
+    {
+        $this->command->info('- CDN fetching files and directories (progress is approximately)...');
+        $bar = $this->command->getOutput()->createProgressBar($expectedMax);
+        $result = $this->edgeStorage->allFiles($edge, fn() => $bar->advance());
+        $bar->finish();
+        $this->command->newLine();
+        return $result;
+    }
 }

app/Bunny/Filesystem/FileSerialized.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace App\Bunny\Filesystem;
+
+interface FileSerialized
+{
+    public function toArray(): array;
+
+    public static function fromArray(array $array): self;
+}

app/Bunny/Filesystem/FileStreamable.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Bunny\Filesystem;
+
+interface FileStreamable
+{
+    /**
+     * @return resource|string
+     */
+    public function getResource();
+}

app/Bunny/Filesystem/LocalFile.php

@@ -4,15 +4,17 @@ namespace App\Bunny\Filesystem;
 
 use Illuminate\Support\Str;
 
-class LocalFile implements File
+class LocalFile implements File, FileStreamable, FileSerialized
 {
     private string $filename;
     private ?string $checksum;
+    private ?string $contents;
 
-    public function __construct(string $filename, ?string $checksum)
+    public function __construct(string $filename, ?string $checksum, string $contents = null)
     {
         $this->filename = $filename;
         $this->checksum = $checksum;
+        $this->contents = $contents;
     }
 
     public function getFilename($search = '', $replace = ''): string
@@ -20,13 +22,35 @@ class LocalFile implements File
         return Str::replaceFirst($search, $replace, $this->filename);
     }
 
-    public function getChecksum(): string
+    public function getChecksum(): ?string
     {
         return $this->checksum;
     }
 
     public function isDirectory(): bool
     {
-        return $this->checksum == null;
+        return $this->getChecksum() === null;
+    }
+
+    public function getResource()
+    {
+        if ($this->contents) {
+            return $this->contents;
+        }
+
+        return fopen($this->getFilename(), 'r');
+    }
+
+    public function toArray(string $search = '', string $replace = ''): array
+    {
+        return [
+            'sha256' => $this->getChecksum(),
+            'filename' => $this->getFilename($search, $replace),
+        ];
+    }
+
+    public static function fromArray(array $array): self
+    {
+        return new self($array['filename'], $array['sha256']);
     }
 }

app/Commands/DeployCommand.php

@@ -2,7 +2,9 @@
 
 namespace App\Commands;
 
+use App\Bunny\Filesystem\CompareOptions;
 use App\Bunny\Filesystem\EdgeStorage;
+use App\Bunny\Filesystem\Exceptions\FilesystemException;
 use App\Bunny\Filesystem\FileCompare;
 use App\Bunny\Filesystem\LocalStorage;
 use Illuminate\Console\Scheduling\Schedule;
@@ -16,7 +18,11 @@ class DeployCommand extends Command
      * @var string
      */
     protected $signature = 'deploy
-        {--dir=dist : Root directory to upload}';
+        {--dir=dist : Root directory to upload}
+        {--no-sha256-cache : Skips .well-known/bunny.sha256 and queries the storage endpoints recursively instead}
+        {--no-sha256-generation : Skips .well-known/bunny.sha256 generation}
+        {--sha256-name=.well-known/bunny.sha256 : Change filename of .well-known/bunny.sha256}
+        {--dry-run : Outputs the operations but will not execute anything}';
 
     /**
      * The description of the command.
@@ -34,9 +40,11 @@ class DeployCommand extends Command
     {
         $start = microtime(true);
 
-        $edgeStorage = new EdgeStorage();
-        $localStorage = new LocalStorage();
-        $fileCompare = new FileCompare($localStorage, $edgeStorage, $this);
+        $fileCompare = new FileCompare(
+            app(LocalStorage::class),
+            app(EdgeStorage::class),
+            $this
+        );
 
         $localPath = realpath($path = $this->option('dir') ?? 'dist');
 
@@ -47,7 +55,23 @@ class DeployCommand extends Command
 
         $edgePath = sprintf('/%s', config('bunny.storage.username'));
 
-        $fileCompare->compare($localPath, $edgePath, $start);
+        if ($this->option('dry-run')) {
+            $this->warn('⚠ Dry run is activated. The operations are displayed but not executed.');
+        }
+
+        try {
+            $fileCompare->compare($localPath, $edgePath, [
+                CompareOptions::START => $start,
+                CompareOptions::NO_SHA256_CACHE => $this->option('no-sha256-cache'),
+                CompareOptions::NO_SHA256_GENERATION => $this->option('no-sha256-generation'),
+                CompareOptions::SHA256_NAME => $this->option('sha256-name'),
+                CompareOptions::DRY_RUN => $this->option('dry-run'),
+            ]);
+        } catch (FilesystemException $exception) {
+            $this->error($exception->getMessage());
+
+            return 2;
+        }
 
         return 0;
     }