Merge pull request #9 from raulp/memory_changes

[WIP] Pre-release changes/fixes

app/Client/Factory.php

@@ -2,21 +2,18 @@
 
 namespace App\Client;
 
+use App\Client\Http\Controllers\ClearLogsController;
 use App\Client\Http\Controllers\CreateTunnelController;
 use App\Client\Http\Controllers\PushLogsToDashboardController;
-use App\Client\Http\HttpClient;
 use App\Http\App;
 use App\Client\Http\Controllers\AttachDataToLogController;
-use App\Client\Http\Controllers\ClearLogsController;
 use App\Client\Http\Controllers\DashboardController;
 use App\Client\Http\Controllers\LogController;
 use App\Client\Http\Controllers\ReplayLogController;
-use App\Http\Controllers\StoreLogController;
 use App\Http\RouteGenerator;
 use App\WebSockets\Socket;
 use Ratchet\WebSocket\WsServer;
 use React\EventLoop\LoopInterface;
-use Symfony\Component\Routing\Route;
 use React\EventLoop\Factory as LoopFactory;
 
 class Factory

app/Client/Http/Controllers/AttachDataToLogController.php

@@ -26,7 +26,7 @@ class AttachDataToLogController extends Controller
         if (! is_null($loggedRequest)) {
             $loggedRequest->setAdditionalData((array)$request->get('data', []));
 
-            $this->requestLogger->pushLogs();
+            $this->requestLogger->pushLoggedRequest($loggedRequest);
 
             $httpConnection->send(str(new Response(200)));
             return;

app/Client/Http/Controllers/DashboardController.php

@@ -17,6 +17,7 @@ class DashboardController extends Controller
     {
         $httpConnection->send(respond_html($this->getView($httpConnection, 'client.dashboard', [
             'subdomains' => Client::$subdomains,
+            'max_logs'=> config()->get('expose.max_logged_requests', 10),
         ])));
     }
 }

app/Logger/CliRequestLogger.php

@@ -45,7 +45,7 @@ class CliRequestLogger extends Logger
         } else {
             $this->requests->prepend($loggedRequest, $loggedRequest->id());
         }
-        $this->requests = $this->requests->slice(0, 10);
+        $this->requests = $this->requests->slice(0, config('expose.max_logged_requests', 10));
 
         $this->section->clear();
 

app/Logger/LoggedRequest.php

@@ -61,7 +61,7 @@ class LoggedRequest implements \JsonSerializable
             'request' => [
                 'raw' => $this->isBinary($this->rawRequest) ? 'BINARY' : $this->rawRequest,
                 'method' => $this->parsedRequest->getMethod(),
-                'uri' => $this->parsedRequest->getUri()->getPath(),
+                'uri' => $this->parsedRequest->getUriString(),
                 'headers' => $this->parsedRequest->getHeaders()->toArray(),
                 'body' => $this->isBinary($this->rawRequest) ? 'BINARY' : $this->parsedRequest->getContent(),
                 'query' => $this->parsedRequest->getQuery()->toArray(),
@@ -72,17 +72,20 @@ class LoggedRequest implements \JsonSerializable
         ];
 
         if ($this->parsedResponse) {
+            $logBody = $this->shouldReturnBody();
+
             try {
-                $body = $this->parsedResponse->getBody();
+                $body = $logBody ? $this->parsedResponse->getBody() : '';
             } catch (\Exception $e) {
                 $body = '';
             }
+
             $data['response'] = [
-                'raw' => $this->shouldReturnBody() ? $this->rawResponse : 'BINARY',
+                'raw' => $logBody ? $this->rawResponse : 'SKIPPED BY CONFIG OR BINARY RESPONSE',
                 'status' => $this->parsedResponse->getStatusCode(),
                 'headers' => $this->parsedResponse->getHeaders()->toArray(),
                 'reason' => $this->parsedResponse->getReasonPhrase(),
-                'body' => $this->shouldReturnBody() ? $body : 'BINARY',
+                'body' => $logBody ? $body : 'SKIPPED BY CONFIG OR BINARY RESPONSE',
             ];
         }
 
@@ -104,11 +107,95 @@ class LoggedRequest implements \JsonSerializable
         return preg_match('~[^\x20-\x7E\t\r\n]~', $string) > 0;
     }
 
-    protected function shouldReturnBody()
+    protected function shouldReturnBody(): bool
     {
-        $contentType = Arr::get($this->parsedResponse->getHeaders()->toArray(), 'Content-Type');
+        if ($this->skipByStatus()) {
+            return false;
+        }
 
-        return $contentType === 'application/json' || Str::is('text/*', $contentType) || Str::is('*javascript*', $contentType);
+        if ($this->skipByContentType()) {
+            return false;
+        }
+
+        if ($this->skipByExtension()) {
+            return false;
+        }
+
+        if ($this->skipBySize()) {
+            return false;
+        }
+
+        $header = $this->parsedResponse->getHeaders()->get('Content-Type');
+        $contentType = $header ? $header->getMediaType() : '';
+        $patterns = [
+            'application/json',
+            'text/*',
+            '*javascript*',
+        ];
+
+
+        return Str::is($patterns, $contentType);
+    }
+
+    protected function skipByStatus(): bool
+    {
+        if (empty(config()->get('expose.skip_body_log.status'))) {
+            return false;
+        }
+
+        return Str::is(config()->get('expose.skip_body_log.status'), $this->parsedResponse->getStatusCode());
+    }
+
+    protected function skipByContentType(): bool
+    {
+        if (empty(config()->get('expose.skip_body_log.content_type'))) {
+            return false;
+        }
+
+        $header = $this->parsedResponse->getHeaders()->get('Content-Type');
+        $contentType = $header ? $header->getMediaType() : '';
+
+        return Str::is(config()->get('expose.skip_body_log.content_type'), $contentType);
+    }
+    
+    protected function skipByExtension(): bool
+    {
+        if (empty(config()->get('expose.skip_body_log.extension'))) {
+            return false;
+        }
+
+        return Str::is(config()->get('expose.skip_body_log.extension'), $this->parsedRequest->getUri()->getPath());
+    }
+
+    protected function skipBySize(): bool
+    {
+        $configSize = $this->getConfigSize(config()->get('expose.skip_body_log.size', '1MB'));
+        $contentLength = $this->parsedResponse->getHeaders()->get('Content-Length');
+
+        if (! $contentLength) {
+            return false;
+        }
+
+        $contentSize = $contentLength->getFieldValue() ?? 0;
+
+        return $contentSize > $configSize;
+    }
+
+    protected function getConfigSize(string $size): int
+    {
+        $units = ['B', 'KB', 'MB', 'GB'];
+        $number = substr($size, 0, -2);
+        $suffix = strtoupper(substr($size,-2));
+
+        // B or no suffix
+        if (is_numeric(substr($suffix, 0, 1))) {
+            return preg_replace('/[^\d]/', '', $size);
+        }
+
+        // if we have an error in the input, default to GB
+        $exponent = array_flip($units)[$suffix] ?? 5;
+
+        return $number * (1024 ** $exponent);
     }
 
     public function getRequest()

app/Logger/RequestLogger.php

@@ -2,6 +2,7 @@
 
 namespace App\Logger;
 
+use App\WebSockets\Socket;
 use Clue\React\Buzz\Browser;
 use GuzzleHttp\RequestOptions;
 use Laminas\Http\Request;
@@ -13,9 +14,6 @@ class RequestLogger
     /** @var array */
     protected $requests = [];
 
-    /** @var array */
-    protected $responses = [];
-
     /** @var CliRequestLogger */
     protected $cliRequestLogger;
 
@@ -42,23 +40,24 @@ class RequestLogger
 
         $this->cliRequestLogger->logRequest($loggedRequest);
 
-        $this->pushLogs();
+        $this->pushLoggedRequest($loggedRequest);
 
         return $loggedRequest;
     }
 
     public function logResponse(Request $request, string $rawResponse)
     {
-        $loggedRequest = collect($this->requests)->first(function (LoggedRequest $loggedRequest) use ($request) {
+        $this->requests = collect($this->requests)->transform(function (LoggedRequest $loggedRequest) use ($request, $rawResponse) {
-            return $loggedRequest->getRequest() === $request;
+            if ($loggedRequest->getRequest() !== $request) {
-        });
+                return $loggedRequest;
-        if ($loggedRequest) {
+            }
+
             $loggedRequest->setResponse($rawResponse, Response::fromString($rawResponse));
-
             $this->cliRequestLogger->logRequest($loggedRequest);
+            $this->pushLoggedRequest($loggedRequest);
 
-            $this->pushLogs();
+            return $loggedRequest;
-        }
+        })->toArray();
     }
 
     public function getData(): array
@@ -69,19 +68,17 @@ class RequestLogger
     public function clear()
     {
         $this->requests = [];
-
-        $this->pushLogs();
     }
 
-    public function pushLogs()
+    public function pushLoggedRequest(LoggedRequest $request)
     {
-        // TODO: Make dashboard part configurable
         $this
             ->client
             ->post(
                 'http://127.0.0.1:4040/api/logs',
                 ['Content-Type' => 'application/json'],
-                json_encode($this->getData(), JSON_INVALID_UTF8_IGNORE)
+                json_encode($request, JSON_INVALID_UTF8_IGNORE)
             );
     }
+
 }

app/Providers/AppServiceProvider.php

@@ -20,6 +20,8 @@ class AppServiceProvider extends ServiceProvider
     {
         $this->loadConfigurationFile();
 
+        $this->setMemoryLimit();
+
         $this->app->singleton(LoopInterface::class, function () {
             return LoopFactory::create();
         });
@@ -53,4 +55,9 @@ class AppServiceProvider extends ServiceProvider
             config()->set('expose', array_merge($builtInConfig, $globalConfig));
         }
     }
+
+    protected function setMemoryLimit()
+    {
+        ini_set('memory_limit', config()->get('expose.memory_limit', '128M'));
+    }
 }

config/expose.php

@@ -66,6 +66,59 @@ return [
     */
     'max_logged_requests' => 25,
 
+    /*
+    |--------------------------------------------------------------------------
+    | Maximum Allowed Memory
+    |--------------------------------------------------------------------------
+    |
+    | The maximum memory allocated to the expose process.
+    |
+    */
+    'memory_limit' => '128M',
+
+    /*
+    |--------------------------------------------------------------------------
+    | Maximum Allowed Memory
+    |--------------------------------------------------------------------------
+    |
+    | Sometimes, some responses don't need to be logged. Some are too big,
+    | some can't be read (like compiled assets). This configuration allows you
+    | to be as granular as you wish when logging the responses.
+    |
+    | If you run constantly out of memory, you probably need to set some of these up.
+    |
+    | Keep in mind, by default, BINARY requests/responses are not logged.
+    | You do not need to add video/mp4 for example to this list.
+    |
+    */
+    'skip_body_log' => [
+        /**
+         | Skip response logging by HTTP response code. Format: 4*, 5*
+         */
+        'status' => [
+            // "4*"
+        ],
+        /**
+         | Skip response logging by HTTP response content type. Ex: "text/css"
+         */
+        'content_type' => [
+            //
+        ],
+        /**
+         | Skip response logging by file extension. Ex: ".js.map", ".min.js", ".min.css"
+         */
+        'extension' => [
+            '.js.map',
+            '.css.map',
+        ],
+        /**
+         | Skip response logging if response size is greater than configured value.
+         | Valid suffixes are: B, KB, MB, GB.
+         | Ex: 500B, 1KB, 2MB, 3GB
+         */
+        'size' => '1MB',
+    ],
+
     'admin' => [
 
         /*

resources/views/client/dashboard.twig

@@ -1,8 +1,11 @@
-<html>
+<html lang="en">
 <head>
+    <title>Expose Dashboard :: {{ subdomains|join(", ") }}</title>
     <script src="https://cdn.jsdelivr.net/npm/vue@2.5.17/dist/vue.js"></script>
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@tailwindcss/ui@latest/dist/tailwind-ui.min.css">
     <script src="https://cdn.jsdelivr.net/npm/clipboard@2/dist/clipboard.min.js"></script>
+    <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@10.1.0/build/styles/github.min.css">
+    <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@10.1.0/build/highlight.min.js" async></script>
     <script>
         !function (a, b) {
             "function" == typeof define && define.amd ? define([], b) : "undefined" != typeof module && module.exports ? module.exports = b() : a.ReconnectingWebSocket = b()
@@ -136,7 +139,7 @@
                         <tr v-for="log in filteredLogs"
                             :class="{'bg-gray-100': currentLog === log}"
                             @click="setLog(log)">
-                            <td class="cursor.pointer px-6 py-4 whitespace-no-wrap border-b border-gray-200 text-sm leading-5 font-medium text-gray-900">
+                            <td class="cursor.pointer px-6 py-4 whitespace-normal border-b border-gray-200 text-sm leading-5 font-medium text-gray-900">
                                 <p>
                                     @{ log.request.method }
                                     @{ log.request.uri }
@@ -396,7 +399,10 @@
                         <pre class="p-6 text-sm whitespace-pre-wrap break-all">@{ currentLog.response.body }</pre>
                     </div>
                     <div v-if="activeTab === 'preview'">
-                        <iframe :srcdoc="currentLog.response.body" style="height: 500px;" class="w-full h-full"></iframe>
+                        <div v-if="responseIsJson()">
+                            <pre><span id="jsonResponseBody" class="json"></span></pre>
+                        </div>
+                        <iframe v-else :srcdoc="currentLog.response.body" style="height: 500px;" class="w-full h-full"></iframe>
                     </div>
                 </div>
             </div>
@@ -421,6 +427,7 @@
             view: 'request',
             activeTab: 'raw',
             logs: [],
+            maxLogs: {{ max_logs }},
         },
 
         computed: {
@@ -437,11 +444,29 @@
         methods: {
             setActiveTab: function(tab) {
                 this.activeTab = tab;
+                this.$nextTick(function () {
+                  this.formatJsonResponse();
+                });
             },
             clearLogs: function() {
                 fetch('/api/logs/clear');
+                this.logs = []
                 this.currentLog = null;
             },
+            formatJsonResponse: function () {
+                if (this.view === 'response' && this.activeTab === 'preview' && this.responseIsJson()) {
+                    const target = document.getElementById('jsonResponseBody');
+                    target.innerText = JSON.stringify(JSON.parse(this.currentLog.response.body), null, 2)
+                    hljs.highlightBlock(target);
+                }
+            },
+            responseIsJson: function() {
+              if (! this.currentLog || ! this.currentLog.response || ! this.currentLog.response.headers) {
+                return false;
+              }
+
+              return /application\/json/g.test(this.currentLog.response.headers['Content-Type']);
+            },
             toPhpArray: function(rows, variableName) {
                 let output = `$${variableName} = [\n`;
 
@@ -461,8 +486,10 @@
             },
             setLog: function (log) {
                 this.currentLog = log;
+                this.formatJsonResponse();
 
                 this.$nextTick(function () {
+                    this.formatJsonResponse()
                     new ClipboardJS('.clipboard');
 
                     new ClipboardJS('.clipboard-query', {
@@ -494,7 +521,15 @@
                 let conn = new ReconnectingWebSocket(`ws://${window.location.hostname}:${window.location.port}/socket`);
 
                 conn.onmessage = (e) => {
-                    this.logs = JSON.parse(e.data);
+                  const request = JSON.parse(e.data);
+                  const index = this.logs.findIndex(log => log.id === request.id);
+                  if (index > -1) {
+                    this.$set(this.logs, index, request)
+                  } else {
+                    this.logs.unshift(request);
+                  }
+
+                  this.logs = this.logs.splice(0, this.maxLogs);
                 };
             },
             loadLogs: function (log) {