bitinflow/bunny-cli
1
0
Fork 0
mirror of https://github.com/bitinflow/bunny-cli.git synced 2026-03-13 13:45:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve uploads/deletions

Browse Source
This commit is contained in:
René Preuß
2021-07-21 19:27:28 +02:00
parent d2767b326b
commit 53cff7d805
8 changed files with 170 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/Bunny/Filesystem/EdgeStorage.php
View File

@@ -6,6 +6,7 @@ use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Promise\PromiseInterface;
use GuzzleHttp\RequestOptions;
use Illuminate\Console\OutputStyle;
use Illuminate\Support\Str;
use Psr\Http\Message\ResponseInterface;
@@ -20,7 +21,7 @@ class EdgeStorage
]);
}
public function allFiles(string $path, &$results = array())
public function allFiles(string $path, callable $advance = null, &$results = array())
{
$promise = $this->client->getAsync(self::normalizePath($path, true), [
RequestOptions::HEADERS => [
@@ -29,7 +30,7 @@ class EdgeStorage
]);
$promise->then(
function (ResponseInterface $res) use (&$results) {
function (ResponseInterface $res) use ($advance, &$results) {
$files = array_map(
fn($file) => new EdgeFile($file),
json_decode($res->getBody()->getContents(), false)
@@ -38,10 +39,13 @@ class EdgeStorage
foreach ($files as $file) {
$results[] = $file;
if ($file->isDirectory()) {
$this->allFiles($file->getFilename(), $results);
$this->allFiles($file->getFilename(), $advance, $results);
}
}
if ($advance) {
$advance();
}
},
function (RequestException $e) {
echo $e->getMessage() . "\n";

113
app/Bunny/Filesystem/FileCompare.php
View File

@@ -14,106 +14,117 @@ class FileCompare
private LocalStorage $localStorage;
private EdgeStorage $edgeStorage;
private Command $output;
private Client $apiClient;
private Command $command;
public function __construct(LocalStorage $localStorage, EdgeStorage $edgeStorage, Command $output)
public function __construct(LocalStorage $localStorage, EdgeStorage $edgeStorage, Command $command)
{
$this->localStorage = $localStorage;
$this->edgeStorage = $edgeStorage;
$this->apiClient = new Client();
$this->output = $output;
$this->command = $command;
}
public function compare(string $local, string $edge): void
public function compare(string $local, string $edge, float $start): void
{
$this->output->info('- Hashing files...');
$localFiles = $this->localStorage->allFiles($local);
$this->output->info(sprintf('✔ Finished hashing %s files', count($localFiles)));
$this->output->info('- CDN diffing files...');
$edgeFiles = $this->edgeStorage->allFiles($edge);
$this->command->info('- Hashing files...');
$localFilesAndDirectories = $this->localStorage->allFiles($local);
$localFiles = array_filter($localFilesAndDirectories, fn(LocalFile $x) => !$x->isDirectory());
$this->command->info(sprintf('✔ Finished hashing %s files', count($localFiles)));
$this->command->info('- CDN fetching files and directories (progress is approximately)...');
$expectedMax = count($localFilesAndDirectories) - count($localFiles);
$bar = $this->command->getOutput()->createProgressBar($expectedMax);
$edgeFiles = $this->edgeStorage->allFiles($edge, fn() => $bar->advance());
$bar->finish();
$this->command->newLine();
$this->command->info(sprintf('✔ Finished fetching %s files and directories', count($edgeFiles)));
$this->command->info('- CDN diffing files and directories...');
$requests = [];
$requestsGroups = ['deletions' => [], 'uploads' => []];
/** @var LocalFile $localFile */
foreach ($localFiles as $localFile) {
if ($localFile->isDirectory()) {
continue;
}
$filename = $localFile->getFilename($local);
if ($match = $this->contains($edgeFiles, $filename, $edge)) {
if ($match->getChecksum() != $localFile->getChecksum()) {
$requests[] = fn() => $this->edgeStorage->put($localFile, $local, $edge);
$requestsGroups['uploads'][] = fn() => $this->edgeStorage->put($localFile, $local, $edge);
}
} else {
$requests[] = fn() => $this->edgeStorage->put($localFile, $local, $edge);
$requestsGroups['uploads'][] = fn() => $this->edgeStorage->put($localFile, $local, $edge);
}
}
/** @var EdgeFile $edgeFile */
foreach ($edgeFiles as $edgeFile) {
$filename = $edgeFile->getFilename($edge);
if (!$this->contains($localFiles, $filename, $local) && !$this->isReserved($filename)) {
$requests[] = fn() => $this->edgeStorage->delete($edgeFile);
if (!$this->contains($localFilesAndDirectories, $filename, $local) && !$this->isReserved($filename)) {
$requestsGroups['deletions'][$filename] = fn() => $this->edgeStorage->delete($edgeFile);
}
}
$this->output->info(sprintf('✔ CDN requesting %s files', $count = count($requests)));
$requestsGroups['deletions'] = Sort::unique($requestsGroups['deletions']);
if ($count > 0) {
$this->output->info(sprintf('- Synchronizing %s files', $count));
$this->command->info('✔ Finished diffing files and directories');
$bar = $this->output->getOutput()->createProgressBar($count);
foreach ($requestsGroups as $type => $requests) {
$operations = count($requests);
if ($operations > 0) {
$this->command->info(sprintf('- CDN requesting %s %s', $operations, $type));
$pool = new Pool($this->edgeStorage->getClient(), $requests, [
'concurrency' => 5,
'fulfilled' => function (Response $response, $index) use ($bar) {
$bar->advance();
},
'rejected' => function (RequestException $reason, $index) use ($bar) {
$bar->advance();
$bar = $this->command->getOutput()->createProgressBar($operations);
if ($this->rejectedDue404Deletion($reason)) {
return;
}
$pool = new Pool($this->edgeStorage->getClient(), $requests, [
'concurrency' => 5,
'fulfilled' => function (Response $response, $index) use ($bar) {
$bar->advance();
},
'rejected' => function (RequestException $reason, $index) use ($bar) {
$bar->advance();
$this->output->warn(sprintf(
'Request rejected by bunny.net. Status: %s, Message: %s',
$reason->getResponse()->getStatusCode(),
$reason->getMessage()
));
},
]);
if ($this->rejectedDue404Deletion($reason)) {
return;
}
// Initiate the transfers and create a promise
$promise = $pool->promise();
$this->command->warn(sprintf(
'Request rejected by bunny.net. Status: %s, Message: %s',
$reason->getResponse()->getStatusCode(),
$reason->getMessage()
));
},
]);
$bar->start();
$promise->wait(); // Force the pool of requests to complete.
$bar->finish();
// Initiate the transfers and create a promise
$promise = $pool->promise();
$this->output->newLine();
$bar->start();
$promise->wait(); // Force the pool of requests to complete.
$bar->finish();
$this->output->info(sprintf('✔ Finished synchronizing %s files', $count));
$this->command->newLine();
$this->command->info(sprintf('✔ Finished synchronizing %s', $type));
}
}
$this->output->info('- Waiting for deploy to go live...');
$this->command->info('- Waiting for deploy to go live...');
$result = $this->apiClient->purgeCache($pullZoneId = config('bunny.pull_zone.id'));
if (!$result->success()) {
$this->output->info('✔ Deploy is live (without flush)!');
$this->command->info('✔ Deploy is live (without flush)!');
return;
}
$result = $this->apiClient->getPullZone($pullZoneId);
$this->output->info('✔ Deploy is live!');
$this->output->newLine();
$timeElapsedSecs = microtime(true) - $start;
$this->command->info(sprintf('✔ Deployment is live! (%ss)', number_format($timeElapsedSecs, 2)));
$this->command->newLine();
foreach ($result->getData()->Hostnames as $hostname) {
$schema = ($hostname->ForceSSL || $hostname->HasCertificate) ? 'https' : 'http';
$this->output->info(sprintf('Website URL: %s://%s', $schema, $hostname->Value));
$this->command->info(sprintf('Website URL: %s://%s', $schema, $hostname->Value));
}
}

39
app/Bunny/Filesystem/Sort.php Normal file
View File

@@ -0,0 +1,39 @@
<?php
namespace App\Bunny\Filesystem;
use Illuminate\Support\Str;
class Sort
{
public static function unique(&$directories): array
{
$relevant = [];
// sort all requested files
uksort($directories, function (string $a, string $b) {
$a = count(explode('/', $a));
$b = count(explode('/', $b));
if ($a == $b) {
return 0;
}
return ($a < $b) ? -1 : 1;
});
// filter all child files and directories
foreach ($directories as $path => $request) {
if (!Str::startsWith($path, array_keys($relevant))) {
$relevant[$path] = $request;
}
}
return $relevant;
}
private static function isParentDeleted(array $parents, string $file): bool
{
return Str::startsWith($file, $parents);
}
}

4
app/Commands/DeployCommand.php
View File

@@ -32,6 +32,8 @@ class DeployCommand extends Command
*/
public function handle(): int
{
$start = microtime(true);
$edgeStorage = new EdgeStorage();
$localStorage = new LocalStorage();
$fileCompare = new FileCompare($localStorage, $edgeStorage, $this);
@@ -45,7 +47,7 @@ class DeployCommand extends Command
$edgePath = sprintf('/%s', config('bunny.storage.username'));
$fileCompare->compare($localPath, $edgePath);
$fileCompare->compare($localPath, $edgePath, $start);
return 0;
}