From 9368f5445dda8b63133c6051aa819f641a0ee61d Mon Sep 17 00:00:00 2001 From: fabrixxm Date: Sat, 27 Mar 2021 18:28:09 +0100 Subject: [PATCH] Display structured logs in admin Tries to parse log lines and to display info in a table. Additional JSON data is parsed and displayed clicking on a row. File reading and line parsing is handled in iterators, to avoid to keep too much data in memory. Search and filter should be trivial to add. Log file is read backward to display log events newest first. A "tail" functionality should be easy to implement. --- src/Model/Log/ParsedLogIterator.php | 81 ++++++++++++++++++++ src/Module/Admin/Logs/View.php | 39 ++++------ src/Object/Log/ParsedLog.php | 114 ++++++++++++++++++++++++++++ src/Util/ReversedFileReader.php | 101 ++++++++++++++++++++++++ view/js/module/admin/logs/view.js | 7 ++ view/templates/admin/logs/view.tpl | 44 ++++++++++- 6 files changed, 361 insertions(+), 25 deletions(-) create mode 100644 src/Model/Log/ParsedLogIterator.php create mode 100644 src/Object/Log/ParsedLog.php create mode 100644 src/Util/ReversedFileReader.php create mode 100644 view/js/module/admin/logs/view.js diff --git a/src/Model/Log/ParsedLogIterator.php b/src/Model/Log/ParsedLogIterator.php new file mode 100644 index 0000000000..621381ac55 --- /dev/null +++ b/src/Model/Log/ParsedLogIterator.php @@ -0,0 +1,81 @@ +. + * + */ +namespace Friendica\Model\Log; + +use \Friendica\Util\ReversedFileReader; +use \Friendica\Object\Log\ParsedLog; + + +/** + * An iterator which returns `\Friendica\Objec\Log\ParsedLog` instances + * + * Uses `\Friendica\Util\ReversedFileReader` to fetch log lines + * from newest to oldest + */ +class ParsedLogIterator implements \Iterator +{ + public function __construct(string $filename, int $limit=0) + { + $this->reader = new ReversedFileReader($filename); + $this->_value = null; + $this->_limit = $limit; + } + + public function next() + { + $this->reader->next(); + if ($this->_limit > 0 && $this->reader->key() > $this->_limit) { + $this->_value = null; + return; + } + if ($this->reader->valid()) { + $line = $this->reader->current(); + $this->_value = new ParsedLog($this->reader->key(), $line); + } else { + $this->_value = null; + } + } + + + public function rewind() + { + $this->_value = null; + $this->reader->rewind(); + $this->next(); + } + + public function key() + { + return $this->reader->key(); + } + + public function current() + { + return $this->_value; + } + + public function valid() + { + return ! is_null($this->_value); + } + +} + diff --git a/src/Module/Admin/Logs/View.php b/src/Module/Admin/Logs/View.php index 91e8f2dd81..339a28b6a5 100644 --- a/src/Module/Admin/Logs/View.php +++ b/src/Module/Admin/Logs/View.php @@ -21,49 +21,42 @@ namespace Friendica\Module\Admin\Logs; -use Friendica\Core\Renderer; use Friendica\DI; +use Friendica\Core\Renderer; +use Friendica\Core\Theme; use Friendica\Module\BaseAdmin; -use Friendica\Util\Strings; +use Friendica\Model\Log\ParsedLogIterator; class View extends BaseAdmin { + const LIMIT = 500; + public static function content(array $parameters = []) { parent::content($parameters); $t = Renderer::getMarkupTemplate('admin/logs/view.tpl'); + DI::page()->registerFooterScript(Theme::getPathForFile('js/module/admin/logs/view.js')); + $f = DI::config()->get('system', 'logfile'); - $data = ''; + $data = null; + $error = null; + if (!file_exists($f)) { - $data = DI::l10n()->t('Error trying to open %1$s log file.\r\n
Check to see if file %1$s exist and is readable.', $f); + $error = DI::l10n()->t('Error trying to open %1$s log file.\r\n
Check to see if file %1$s exist and is readable.', $f); } else { - $fp = fopen($f, 'r'); - if (!$fp) { - $data = DI::l10n()->t('Couldn\'t open %1$s log file.\r\n
Check to see if file %1$s is readable.', $f); - } else { - $fstat = fstat($fp); - $size = $fstat['size']; - if ($size != 0) { - if ($size > 5000000 || $size < 0) { - $size = 5000000; - } - $seek = fseek($fp, 0 - $size, SEEK_END); - if ($seek === 0) { - $data = Strings::escapeHtml(fread($fp, $size)); - while (!feof($fp)) { - $data .= Strings::escapeHtml(fread($fp, 4096)); - } - } - } - fclose($fp); + try { + $data = new ParsedLogIterator($f, self::LIMIT); + } catch (Exception $e) { + $error = DI::l10n()->t('Couldn\'t open %1$s log file.\r\n
Check to see if file %1$s is readable.', $f); } } return Renderer::replaceMacros($t, [ '$title' => DI::l10n()->t('Administration'), '$page' => DI::l10n()->t('View Logs'), '$data' => $data, + '$error' => $error, '$logname' => DI::config()->get('system', 'logfile') ]); } diff --git a/src/Object/Log/ParsedLog.php b/src/Object/Log/ParsedLog.php new file mode 100644 index 0000000000..21bd538765 --- /dev/null +++ b/src/Object/Log/ParsedLog.php @@ -0,0 +1,114 @@ +. + * + */ +namespace Friendica\Object\Log; + +/** + * Parse a log line and offer some utility methods + */ +class ParsedLog +{ + const REGEXP = '/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^ ]*) (\w+) \[(\w*)\]: (.*)/'; + + public $id = 0; + public $date = null; + public $context = null; + public $level = null; + public $message = null; + public $data = null; + public $source = null; + + /** + * @param string $logline Source log line to parse + */ + public function __construct(int $id, string $logline) + { + $this->id = $id; + $this->parse($logline); + $this->stop = false; + } + + private function parse($logline) + { + list($logline, $jsonsource) = explode(' - ', $logline); + $jsondata = null; + if (strpos($logline, '{"') > 0) { + list($logline, $jsondata) = explode('{"', $logline, 2); + $jsondata = '{"' . $jsondata; + } + preg_match(self::REGEXP, $logline, $matches); + $this->date = $matches[1]; + $this->context = $matches[2]; + $this->level = $matches[3]; + $this->message = $matches[4]; + $this->data = $jsondata; + $this->source = $jsonsource; + $this->try_fix_json('data'); + } + + /** + * In log boundary between message and json data is not specified. + * If message contains '{' the parser thinks there starts the json data. + * This method try to parse the found json and if it fails, search for next '{' + * in json data and retry + */ + private function try_fix_json(string $key) + { + if (is_null($this->$key) || $this->$key == "") { + return; + } + try { + $d = json_decode($this->$key, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + // try to find next { in $str and move string before to 'message' + + $pos = strpos($this->$key, '{', 1); + + $this->message .= substr($this->$key, 0, $pos); + $this->$key = substr($this->key, $pos); + $this->try_fix_json($key); + } + } + + /** + * Return decoded `data` as array suitable for template + * + * @return array + */ + public function get_data() { + $data = json_decode($this->data, true); + if ($data) { + foreach($data as $k => $v) { + $v = print_r($v, true); + $data[$k] = $v; + } + } + return $data; + } + + /** + * Return decoded `source` as array suitable for template + * + * @return array + */ + public function get_source() { + return json_decode($this->source, true); + } +} diff --git a/src/Util/ReversedFileReader.php b/src/Util/ReversedFileReader.php new file mode 100644 index 0000000000..8a3083f7d8 --- /dev/null +++ b/src/Util/ReversedFileReader.php @@ -0,0 +1,101 @@ +. + * + */ + +namespace Friendica\Util; + + +/** + * An iterator which returns lines from file in reversed order + * + * original code https://stackoverflow.com/a/10494801 + */ +class ReversedFileReader implements \Iterator +{ + const BUFFER_SIZE = 4096; + const SEPARATOR = "\n"; + + public function __construct($filename) + { + $this->_fh = fopen($filename, 'r'); + if (!$this->_fh) { + // this should use a custom exception. + throw \Exception("Unable to open $filename"); + } + $this->_filesize = filesize($filename); + $this->_pos = -1; + $this->_buffer = null; + $this->_key = -1; + $this->_value = null; + } + + public function _read($size) + { + $this->_pos -= $size; + fseek($this->_fh, $this->_pos); + return fread($this->_fh, $size); + } + + public function _readline() + { + $buffer =& $this->_buffer; + while (true) { + if ($this->_pos == 0) { + return array_pop($buffer); + } + if (count($buffer) > 1) { + return array_pop($buffer); + } + $buffer = explode(self::SEPARATOR, $this->_read(self::BUFFER_SIZE) . $buffer[0]); + } + } + + public function next() + { + ++$this->_key; + $this->_value = $this->_readline(); + } + + public function rewind() + { + if ($this->_filesize > 0) { + $this->_pos = $this->_filesize; + $this->_value = null; + $this->_key = -1; + $this->_buffer = explode(self::SEPARATOR, $this->_read($this->_filesize % self::BUFFER_SIZE ?: self::BUFFER_SIZE)); + $this->next(); + } + } + + public function key() + { + return $this->_key; + } + + public function current() + { + return $this->_value; + } + + public function valid() + { + return ! is_null($this->_value); + } +} diff --git a/view/js/module/admin/logs/view.js b/view/js/module/admin/logs/view.js new file mode 100644 index 0000000000..45d08a5ed1 --- /dev/null +++ b/view/js/module/admin/logs/view.js @@ -0,0 +1,7 @@ +function log_show_details(id) { + document + .querySelectorAll('[data-id="' + id + '"]') + .forEach(elm => { + elm.classList.toggle('hidden') + }); +} diff --git a/view/templates/admin/logs/view.tpl b/view/templates/admin/logs/view.tpl index 9ac5acd9dd..166dea0ba9 100644 --- a/view/templates/admin/logs/view.tpl +++ b/view/templates/admin/logs/view.tpl @@ -1,6 +1,46 @@

{{$title}} - {{$page}}

- +

{{$logname}}

- 
{{$data}}
+ {{if $error }} +
+

{{$error nofilter}}

+
+ {{else}} + + + + + + + + + + + {{foreach $data as $row}} + + + + + + + {{foreach $row->get_data() as $k=>$v}} + + + + + {{/foreach}} + + {{foreach $row->get_source() as $k=>$v}} + + + + + {{/foreach}} + {{/foreach}} + +
DateLevelContextMessage
{{$row->date}}{{$row->level}}{{$row->context}}{{$row->message}}
+ {{/if}}