From 23c4543773509422522759e9c075472a0d4d77cd Mon Sep 17 00:00:00 2001 From: Raul Predescu Date: Sun, 14 Jun 2020 14:36:59 -0700 Subject: [PATCH] pre-release changes --- app/Client/Factory.php | 5 +- .../Controllers/AttachDataToLogController.php | 2 +- .../Http/Controllers/ClearLogsController.php | 2 +- .../Http/Controllers/DashboardController.php | 1 + app/Commands/ServeCommand.php | 2 +- app/Commands/ShareCommand.php | 2 +- app/Logger/CliRequestLogger.php | 2 +- app/Logger/LoggedRequest.php | 101 ++++++++++++++++-- app/Logger/RequestLogger.php | 29 +++-- app/Providers/AppServiceProvider.php | 7 ++ config/expose.php | 53 +++++++++ resources/views/client/dashboard.twig | 43 +++++++- 12 files changed, 213 insertions(+), 36 deletions(-) diff --git a/app/Client/Factory.php b/app/Client/Factory.php index 18341de..2f56d5a 100644 --- a/app/Client/Factory.php +++ b/app/Client/Factory.php @@ -2,21 +2,18 @@ namespace App\Client; +use App\Client\Http\Controllers\ClearLogsController; use App\Client\Http\Controllers\CreateTunnelController; use App\Client\Http\Controllers\PushLogsToDashboardController; -use App\Client\Http\HttpClient; use App\Http\App; use App\Client\Http\Controllers\AttachDataToLogController; -use App\Client\Http\Controllers\ClearLogsController; use App\Client\Http\Controllers\DashboardController; use App\Client\Http\Controllers\LogController; use App\Client\Http\Controllers\ReplayLogController; -use App\Http\Controllers\StoreLogController; use App\Http\RouteGenerator; use App\WebSockets\Socket; use Ratchet\WebSocket\WsServer; use React\EventLoop\LoopInterface; -use Symfony\Component\Routing\Route; use React\EventLoop\Factory as LoopFactory; class Factory diff --git a/app/Client/Http/Controllers/AttachDataToLogController.php b/app/Client/Http/Controllers/AttachDataToLogController.php index f9096fd..416dacb 100644 --- a/app/Client/Http/Controllers/AttachDataToLogController.php +++ b/app/Client/Http/Controllers/AttachDataToLogController.php @@ -26,7 +26,7 @@ class AttachDataToLogController extends Controller if (! is_null($loggedRequest)) { $loggedRequest->setAdditionalData((array)$request->get('data', [])); - $this->requestLogger->pushLogs(); + $this->requestLogger->pushLoggedRequest($loggedRequest); $httpConnection->send(str(new Response(200))); return; diff --git a/app/Client/Http/Controllers/ClearLogsController.php b/app/Client/Http/Controllers/ClearLogsController.php index 641dc0e..69be1c8 100644 --- a/app/Client/Http/Controllers/ClearLogsController.php +++ b/app/Client/Http/Controllers/ClearLogsController.php @@ -23,4 +23,4 @@ class ClearLogsController extends Controller $httpConnection->send(respond_json([], 200)); } -} +} \ No newline at end of file diff --git a/app/Client/Http/Controllers/DashboardController.php b/app/Client/Http/Controllers/DashboardController.php index 8986fb4..eb5ab54 100644 --- a/app/Client/Http/Controllers/DashboardController.php +++ b/app/Client/Http/Controllers/DashboardController.php @@ -17,6 +17,7 @@ class DashboardController extends Controller { $httpConnection->send(respond_html($this->getView($httpConnection, 'client.dashboard', [ 'subdomains' => Client::$subdomains, + 'max_logs'=> config()->get('expose.max_logged_requests', 10), ]))); } } diff --git a/app/Commands/ServeCommand.php b/app/Commands/ServeCommand.php index 1a73cf3..a6ba674 100644 --- a/app/Commands/ServeCommand.php +++ b/app/Commands/ServeCommand.php @@ -10,7 +10,7 @@ class ServeCommand extends Command { protected $signature = 'serve {hostname=localhost} {host=0.0.0.0} {--validateAuthTokens} {--port=8080}'; - protected $description = 'Start the expose server'; + protected $description = 'Start the shaft server'; public function handle() { diff --git a/app/Commands/ShareCommand.php b/app/Commands/ShareCommand.php index 69bcb29..f59afc7 100644 --- a/app/Commands/ShareCommand.php +++ b/app/Commands/ShareCommand.php @@ -13,7 +13,7 @@ class ShareCommand extends Command { protected $signature = 'share {host} {--subdomain=} {--auth=}'; - protected $description = 'Share a local url with a remote expose server'; + protected $description = 'Share a local url with a remote shaft server'; protected function configureConnectionLogger() { diff --git a/app/Logger/CliRequestLogger.php b/app/Logger/CliRequestLogger.php index 3eab8ff..c01a003 100644 --- a/app/Logger/CliRequestLogger.php +++ b/app/Logger/CliRequestLogger.php @@ -45,7 +45,7 @@ class CliRequestLogger extends Logger } else { $this->requests->prepend($loggedRequest, $loggedRequest->id()); } - $this->requests = $this->requests->slice(0, 10); + $this->requests = $this->requests->slice(0, config('expose.max_logged_requests', 10)); $this->section->clear(); diff --git a/app/Logger/LoggedRequest.php b/app/Logger/LoggedRequest.php index 2415312..cc83be4 100644 --- a/app/Logger/LoggedRequest.php +++ b/app/Logger/LoggedRequest.php @@ -61,7 +61,7 @@ class LoggedRequest implements \JsonSerializable 'request' => [ 'raw' => $this->isBinary($this->rawRequest) ? 'BINARY' : $this->rawRequest, 'method' => $this->parsedRequest->getMethod(), - 'uri' => $this->parsedRequest->getUri()->getPath(), + 'uri' => $this->parsedRequest->getUriString(), 'headers' => $this->parsedRequest->getHeaders()->toArray(), 'body' => $this->isBinary($this->rawRequest) ? 'BINARY' : $this->parsedRequest->getContent(), 'query' => $this->parsedRequest->getQuery()->toArray(), @@ -72,17 +72,20 @@ class LoggedRequest implements \JsonSerializable ]; if ($this->parsedResponse) { + $logBody = $this->shouldReturnBody(); + try { - $body = $this->parsedResponse->getBody(); + $body = $logBody ? $this->parsedResponse->getBody() : ''; } catch (\Exception $e) { $body = ''; } + $data['response'] = [ - 'raw' => $this->shouldReturnBody() ? $this->rawResponse : 'BINARY', + 'raw' => $logBody ? $this->rawResponse : 'SKIPPED BY CONFIG OR BINARY RESPONSE', 'status' => $this->parsedResponse->getStatusCode(), 'headers' => $this->parsedResponse->getHeaders()->toArray(), 'reason' => $this->parsedResponse->getReasonPhrase(), - 'body' => $this->shouldReturnBody() ? $body : 'BINARY', + 'body' => $logBody ? $body : 'SKIPPED BY CONFIG OR BINARY RESPONSE', ]; } @@ -104,11 +107,95 @@ class LoggedRequest implements \JsonSerializable return preg_match('~[^\x20-\x7E\t\r\n]~', $string) > 0; } - protected function shouldReturnBody() + protected function shouldReturnBody(): bool { - $contentType = Arr::get($this->parsedResponse->getHeaders()->toArray(), 'Content-Type'); + if ($this->skipByStatus()) { + return false; + } - return $contentType === 'application/json' || Str::is('text/*', $contentType) || Str::is('*javascript*', $contentType); + if ($this->skipByContentType()) { + return false; + } + + if ($this->skipByExtension()) { + return false; + } + + if ($this->skipBySize()) { + return false; + } + + $header = $this->parsedResponse->getHeaders()->get('Content-Type'); + $contentType = $header ? $header->getMediaType() : ''; + $patterns = [ + 'application/json', + 'text/*', + '*javascript*', + ]; + + + return Str::is($patterns, $contentType); + } + + protected function skipByStatus(): bool + { + if (empty(config()->get('expose.skip_body_log.status'))) { + return false; + } + + return Str::is(config()->get('expose.skip_body_log.status'), $this->parsedResponse->getStatusCode()); + } + + protected function skipByContentType(): bool + { + if (empty(config()->get('expose.skip_body_log.content_type'))) { + return false; + } + + $header = $this->parsedResponse->getHeaders()->get('Content-Type'); + $contentType = $header ? $header->getMediaType() : ''; + + return Str::is(config()->get('expose.skip_body_log.content_type'), $contentType); + } + + protected function skipByExtension(): bool + { + if (empty(config()->get('expose.skip_body_log.extension'))) { + return false; + } + + return Str::is(config()->get('expose.skip_body_log.extension'), $this->parsedRequest->getUri()->getPath()); + } + + protected function skipBySize(): bool + { + $configSize = $this->getConfigSize(config()->get('expose.skip_body_log.size', '1MB')); + $contentLength = $this->parsedResponse->getHeaders()->get('Content-Length'); + + if (! $contentLength) { + return false; + } + + $contentSize = $contentLength->getFieldValue() ?? 0; + + return $contentSize > $configSize; + } + + protected function getConfigSize(string $size): int + { + $units = ['B', 'KB', 'MB', 'GB']; + $number = substr($size, 0, -2); + $suffix = strtoupper(substr($size,-2)); + + // B or no suffix + if (is_numeric(substr($suffix, 0, 1))) { + return preg_replace('/[^\d]/', '', $size); + } + + // if we have an error in the input, default to GB + $exponent = array_flip($units)[$suffix] ?? 5; + + return $number * (1024 ** $exponent); } public function getRequest() diff --git a/app/Logger/RequestLogger.php b/app/Logger/RequestLogger.php index 80994de..7a71a38 100644 --- a/app/Logger/RequestLogger.php +++ b/app/Logger/RequestLogger.php @@ -2,6 +2,7 @@ namespace App\Logger; +use App\WebSockets\Socket; use Clue\React\Buzz\Browser; use GuzzleHttp\RequestOptions; use Laminas\Http\Request; @@ -13,9 +14,6 @@ class RequestLogger /** @var array */ protected $requests = []; - /** @var array */ - protected $responses = []; - /** @var CliRequestLogger */ protected $cliRequestLogger; @@ -42,23 +40,24 @@ class RequestLogger $this->cliRequestLogger->logRequest($loggedRequest); - $this->pushLogs(); + $this->pushLoggedRequest($loggedRequest); return $loggedRequest; } public function logResponse(Request $request, string $rawResponse) { - $loggedRequest = collect($this->requests)->first(function (LoggedRequest $loggedRequest) use ($request) { - return $loggedRequest->getRequest() === $request; - }); - if ($loggedRequest) { + $this->requests = collect($this->requests)->transform(function (LoggedRequest $loggedRequest) use ($request, $rawResponse) { + if ($loggedRequest->getRequest() !== $request) { + return $loggedRequest; + } + $loggedRequest->setResponse($rawResponse, Response::fromString($rawResponse)); - $this->cliRequestLogger->logRequest($loggedRequest); + $this->pushLoggedRequest($loggedRequest); - $this->pushLogs(); - } + return $loggedRequest; + })->toArray(); } public function getData(): array @@ -69,19 +68,17 @@ class RequestLogger public function clear() { $this->requests = []; - - $this->pushLogs(); } - public function pushLogs() + public function pushLoggedRequest(LoggedRequest $request) { - // TODO: Make dashboard part configurable $this ->client ->post( 'http://127.0.0.1:4040/api/logs', ['Content-Type' => 'application/json'], - json_encode($this->getData(), JSON_INVALID_UTF8_IGNORE) + json_encode($request, JSON_INVALID_UTF8_IGNORE) ); } + } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 3508f93..7167c79 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -20,6 +20,8 @@ class AppServiceProvider extends ServiceProvider { $this->loadConfigurationFile(); + $this->setMemoryLimit(); + $this->app->singleton(LoopInterface::class, function () { return LoopFactory::create(); }); @@ -53,4 +55,9 @@ class AppServiceProvider extends ServiceProvider config()->set('expose', array_merge($builtInConfig, $globalConfig)); } } + + protected function setMemoryLimit() + { + ini_set('memory_limit', config()->get('expose.memory_limit', '128M')); + } } diff --git a/config/expose.php b/config/expose.php index f8b18f7..8de21d4 100644 --- a/config/expose.php +++ b/config/expose.php @@ -66,6 +66,59 @@ return [ */ 'max_logged_requests' => 25, + /* + |-------------------------------------------------------------------------- + | Maximum Allowed Memory + |-------------------------------------------------------------------------- + | + | The maximum memory allocated to the expose process. + | + */ + 'memory_limit' => '128M', + + /* + |-------------------------------------------------------------------------- + | Maximum Allowed Memory + |-------------------------------------------------------------------------- + | + | Sometimes, some responses don't need to be logged. Some are too big, + | some can't be read (like compiled assets). This configuration allows you + | to be as granular as you wish when logging the responses. + | + | If you run constantly out of memory, you probably need to set some of these up. + | + | Keep in mind, by default, BINARY requests/responses are not logged. + | You do not need to add video/mp4 for example to this list. + | + */ + 'skip_body_log' => [ + /** + | Skip response logging by HTTP response code. Format: 4*, 5* + */ + 'status' => [ + // "4*" + ], + /** + | Skip response logging by HTTP response content type. Ex: "text/css" + */ + 'content_type' => [ + // + ], + /** + | Skip response logging by file extension. Ex: ".js.map", ".min.js", ".min.css" + */ + 'extension' => [ + '.js.map', + '.css.map', + ], + /** + | Skip response logging if response size is greater than configured value. + | Valid suffixes are: B, KB, MB, GB. + | Ex: 500B, 1KB, 2MB, 3GB + */ + 'size' => '1MB', + ], + 'admin' => [ /* diff --git a/resources/views/client/dashboard.twig b/resources/views/client/dashboard.twig index 32b40f4..c76422a 100644 --- a/resources/views/client/dashboard.twig +++ b/resources/views/client/dashboard.twig @@ -1,8 +1,11 @@ - + + Expose Dashboard :: {{ subdomains|join(", ") }} + +