From ff53b1d6ce8593763445ca4927dd361707ddddca Mon Sep 17 00:00:00 2001
From: Carsten Rose <carsten.rose@math.uzh.ch>
Date: Mon, 18 Jun 2018 00:50:29 +0200
Subject: [PATCH] F5458 Error Log Unique Identifier: Exception message might be
an ordinary string or a JSON encoded array.
---
doc/NewVersion.md | 5 +
extension/Documentation/Manual.rst | 10 +-
extension/ext_conf_template.txt | 3 +
extension/qfq/qfq/Constants.php | 15 ++-
extension/qfq/qfq/database/Database.php | 46 ++++++-
.../qfq/qfq/exceptions/AbstractException.php | 112 +++++++++++++-----
extension/qfq/qfq/helper/OnArray.php | 44 ++++++-
extension/qfq/qfq/store/Config.php | 1 +
8 files changed, 190 insertions(+), 46 deletions(-)
diff --git a/doc/NewVersion.md b/doc/NewVersion.md
index e232462f7..e4a711be6 100644
--- a/doc/NewVersion.md
+++ b/doc/NewVersion.md
@@ -62,4 +62,9 @@ https://docs.typo3.org/typo3cms/drafts/github/T3DocumentationStarter/Public-Info
make update-qfq-doc
+Upload new version to TER
+=========================
+* https://extensions.typo3.org/ > Log in > My Extensions.
+* Rename the ZIP file to be TER compatible: e.g. qfq_18.6.0.zip.
+* Upload - that's all.
\ No newline at end of file
diff --git a/extension/Documentation/Manual.rst b/extension/Documentation/Manual.rst
index c2499a1c9..be5eb2657 100644
--- a/extension/Documentation/Manual.rst
+++ b/extension/Documentation/Manual.rst
@@ -262,11 +262,15 @@ Setup a *report* to manage all *forms*:
Install Check List
------------------
-* Protect the directory `<T3 installation>/fileadmin/protected` in Apache against direct file access. Those directory
- should be used for confidential (uploaded / generated) data.
+* Protect the directory `<T3 installation>/fileadmin/protected` in Apache against direct file access.
+
+ * `<T3 installation>/fileadmin/protected/` should be used for confidential (uploaded / generated) data.
+ * `<T3 installation>/fileadmin/protected/log/...` is the default place for QFQ logfiles.
+
* Protect the directory `<T3 installation>/fileadmin` in Apache to not execute PHP Scripts - malicious uploads won't be executed.
* Setup a log rotation rule for `sqlLog`.
-* Check that `sqlLogMode` is set to `modify` on productive sites.
+* Check that `sqlLogMode` is set to `modify` on productive sites. With `none` you have no chance to find out who changed
+ which data and `all` really logs a mass of data.
.. _configuration:
diff --git a/extension/ext_conf_template.txt b/extension/ext_conf_template.txt
index a4bad3791..beda65e0d 100644
--- a/extension/ext_conf_template.txt
+++ b/extension/ext_conf_template.txt
@@ -48,6 +48,9 @@ fillStoreSystemBySqlErrorMsg3 =
+# cat=debug/qfq; type=string; label=QFQ log file:Default is 'fileadmin/protected/log/qfq.log'. A logfile of fired SQL statements. PathFile is absolute or relative to '<site path>'.
+qfqLog = fileadmin/protected/log/qfq.log
+
# cat=debug/sql; type=string; label=SQL log mode:Default is 'modify'. A logfile of QFQ fired SQL statements will be written. Possible modes are 'all' - every statement will be logged (this might a lot). 'modify' - log only statements who change data. 'error' - log only DB errors. 'none' - log never.
sqlLogMode = modify
diff --git a/extension/qfq/qfq/Constants.php b/extension/qfq/qfq/Constants.php
index 94c320305..a30c769c2 100644
--- a/extension/qfq/qfq/Constants.php
+++ b/extension/qfq/qfq/Constants.php
@@ -111,6 +111,10 @@ const KVP_IF_VALUE_EMPTY_COPY_KEY = 'if_value_empty_copy_key';
const KVP_VALUE_GIVEN = 'value_given';
// https://lib2.colostate.edu/wildlife/atoz.php?letter=ALL
+// JSON encoded messages thrown through an exception:
+const ERROR_MESSAGE_TO_USER = 'toUser'; // always shown to the user.
+const ERROR_MESSAGE_SUPPORT = 'support'; // Message to help the developer to understand the problem.
+const ERROR_MESSAGE_OS = 'os'; // Error message from the OS - like 'file not found' or specific SQL problem
// QFQ Error Codes
const ERROR_UNKNOW_SANITIZE_CLASS = 1001;
@@ -174,7 +178,7 @@ const ERROR_MISSING_OPEN_DELIMITER = 1060;
const ERROR_MISSING_CLOSE_DELIMITER = 1061;
const ERROR_EXPECTED_ARRAY = 1062;
const ERROR_REPORT_FAILED_ACTION = 1063;
-const ERROR_MISSING_MESSAGE_FAIL = 1064;
+
const ERROR_MISSING_TABLE_NAME = 1065;
const ERROR_MISSING_TABLE = 1066;
const ERROR_RECORD_NOT_FOUND = 1067;
@@ -403,13 +407,17 @@ const SYSTEM_DB_INDEX_QFQ = "indexQfq";
const SYSTEM_DB_NAME_DATA = '_dbNameData';
const SYSTEM_DB_NAME_QFQ = '_dbNameQfq';
+const SYSTEM_QFQ_LOG = 'qfqLog'; // Logging to file
+const SYSTEM_QFQ_LOG_FILE = 'fileadmin/protected/log/qfq.log';
+
const SYSTEM_SQL_LOG = 'sqlLog'; // Logging to file
-const SYSTEM_SQL_LOG_FILE = '../../sql.log';
+const SYSTEM_SQL_LOG_FILE = 'fileadmin/protected/log/sql.log';
const SYSTEM_SQL_LOG_MODE = 'sqlLogMode'; // Mode, which statements to log.
+
const SYSTEM_DATE_FORMAT = 'dateFormat';
const SYSTEM_REDIRECT_ALL_MAIL_TO = 'redirectAllMailTo';
const SYSTEM_MAIL_LOG = 'mailLog';
-const SYSTEM_MAIL_LOG_FILE = '../../mail.log';
+const SYSTEM_MAIL_LOG_FILE = 'fileadmin/protected/log/mail.log';
const SYSTEM_SHOW_DEBUG_INFO = 'showDebugInfo';
const SYSTEM_SHOW_DEBUG_INFO_YES = 'yes';
@@ -1452,6 +1460,7 @@ const EXCEPTION_TT_CONTENT_UID = 'Content Id';
const EXCEPTION_EDIT_FORM = 'Edit';
const EXCEPTION_TIMESTAMP = 'Timestamp';
+const EXCEPTION_UNIQID = 'UniqId';
const EXCEPTION_CODE = 'Code';
const EXCEPTION_MESSAGE = 'Message'; // Will be shown on every exception. NO sensitive data here!
const EXCEPTION_MESSAGE_DEBUG = SYSTEM_MESSAGE_DEBUG; // Will only be shown as debugging (Typically BE user is logged in)
diff --git a/extension/qfq/qfq/database/Database.php b/extension/qfq/qfq/database/Database.php
index babf2684c..f296188fb 100644
--- a/extension/qfq/qfq/database/Database.php
+++ b/extension/qfq/qfq/database/Database.php
@@ -67,6 +67,7 @@ class Database {
* @throws CodeException
* @throws DbException
* @throws UserFormException
+ * @throws UserReportException
*/
public function __construct($dbIndex = DB_INDEX_DEFAULT) {
if (empty($dbIndex)) {
@@ -275,6 +276,25 @@ class Database {
$this->mysqli_result = null;
}
+ /**
+ * Checks the problematic $sql if there is a common mistake.
+ * If something is found, give a hint.
+ *
+ * @param $sql
+ * @return string
+ */
+ private function getSqlHint($sql){
+
+ // Check if there is a comma before FROM: 'SELECT ... , FROM ...'
+ $pos=stripos($sql, ' FROM ');
+ if($pos!==false && $pos>0 && $sql[$pos -1]==','){
+ return 'HINT: remove extra "," before FROM';
+ }
+
+
+ return '';
+ }
+
/**
* Execute a prepared SQL statement like SELECT, INSERT, UPDATE, DELETE, SHOW, ...
*
@@ -298,32 +318,43 @@ class Database {
$sqlLogMode = $this->isSqlModify($sql) ? SQL_LOG_MODE_MODIFY : SQL_LOG_MODE_ALL;
$result = 0;
$stat = array();
+ $errorMsg[ERROR_MESSAGE_TO_USER] = empty($specificMessage) ? 'SQL error' : $specificMessage;
if ($this->store !== null) {
$this->store->setVar(SYSTEM_SQL_FINAL, $sql, STORE_SYSTEM);
$this->store->setVar(SYSTEM_SQL_PARAM_ARRAY, $parameterArray, STORE_SYSTEM);
}
- if ($specificMessage !== '') {
- $specificMessage = ' - ' . $specificMessage;
- }
+
+// if ($specificMessage !== '') {
+// $specificMessage = ' - ' . $specificMessage;
+// }
// Logfile
$this->dbLog($sqlLogMode, $sql, $parameterArray);
if (false === ($this->mysqli_stmt = $this->mysqli->prepare($sql))) {
$this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
- throw new DbException('[ mysqli: ' . $this->mysqli->errno . ' ] ' . $this->mysqli->error . $specificMessage, ERROR_DB_PREPARE);
+ $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+ $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' . $this->mysqli->errno . ' ] ' . $this->mysqli->error;
+
+ throw new DbException(json_encode($errorMsg), ERROR_DB_PREPARE);
}
if (count($parameterArray) > 0) {
if (false === $this->prepareBindParam($parameterArray)) {
$this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
- throw new DbException('[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error . $specificMessage, ERROR_DB_BIND);
+ $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+ $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error;
+
+ throw new DbException(json_encode($errorMsg), ERROR_DB_BIND);
}
}
if (false === $this->mysqli_stmt->execute()) {
$this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
- throw new DbException('[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error . $specificMessage, ERROR_DB_EXECUTE);
+ $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+ $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error;
+
+ throw new DbException(json_encode($errorMsg),ERROR_DB_EXECUTE);
}
$msg = '';
@@ -595,6 +626,9 @@ class Database {
* @param string $columnName name of the column
*
* @return array
+ * @throws CodeException
+ * @throws DbException
+ * @throws UserFormException
*/
public function getEnumSetValueList($table, $columnName) {
diff --git a/extension/qfq/qfq/exceptions/AbstractException.php b/extension/qfq/qfq/exceptions/AbstractException.php
index baee036b4..62c66f3d6 100644
--- a/extension/qfq/qfq/exceptions/AbstractException.php
+++ b/extension/qfq/qfq/exceptions/AbstractException.php
@@ -16,6 +16,7 @@ require_once(__DIR__ . '/../report/Link.php');
require_once(__DIR__ . '/../database/Database.php');
require_once(__DIR__ . '/UserFormException.php');
require_once(__DIR__ . '/../helper/OnArray.php');
+require_once(__DIR__ . '/../helper/Logger.php');
/**
@@ -33,14 +34,22 @@ class AbstractException extends \Exception {
protected $line = '';
/**
+ * There are 3+1 different messages:
+ * 'toUser' - shown in the client to the user - no details here!!!
+ * 'support' - help for the developer
+ * 'os' - message from the OS, like 'file not found'
+ * Stacktrace, Form, FormElement, Report level, T3 page, T3 tt_content uid, ...
+ *
* @return string
* @throws CodeException
* @throws UserFormException
*/
public function formatException() {
+
$t3Vars = array();
$arrShow = $this->messageArray;
$htmlDebug = '';
+ $arrDebugShow = array();
try {
// In a very early stage, it might be possible that Store can't be initialized: take care not to use it.
@@ -48,7 +57,7 @@ class AbstractException extends \Exception {
$t3Vars = $store->getStore(STORE_TYPO3);
} catch (CodeException $e) {
$store = null;
- } catch (\Exception $exception){
+ } catch (\Exception $exception) {
$store = null;
}
@@ -58,18 +67,32 @@ class AbstractException extends \Exception {
$arrShow[EXCEPTION_TIMESTAMP] = date('Y.m.d H:i:s O');
$arrShow[EXCEPTION_CODE] = $this->getCode();
- $arrShow[EXCEPTION_MESSAGE] = $this->getMessage();
-
- foreach ([EXCEPTION_FORM, EXCEPTION_FORM_ELEMENT, EXCEPTION_FORM_ELEMENT_COLUMN] as $key) {
- if (isset($arrShow[$key])) {
- if ($arrShow[$key] == '') {
- unset($arrShow[$key]);
- }
- }
+ $arrShow[EXCEPTION_UNIQID] = uniqid();
+
+ // Get exception message and if JSON, decode it.
+ $msg = $this->getMessage();
+ $arrMsg = json_decode($msg, true);
+ if ($arrMsg === null) {
+ $arrShow[EXCEPTION_MESSAGE] = $msg;
+ $arrMsg[ERROR_MESSAGE_TO_USER] = $msg;
+ } else {
+ $arrShow[EXCEPTION_MESSAGE] = $arrMsg[ERROR_MESSAGE_TO_USER];
}
- // Debug Information
- if ($store !== null && Support::findInSet(SYSTEM_SHOW_DEBUG_INFO_YES, $store->getVar(SYSTEM_SHOW_DEBUG_INFO, STORE_SYSTEM))) {
+// // Unset empty elements
+// foreach ([EXCEPTION_FORM, EXCEPTION_FORM_ELEMENT, EXCEPTION_FORM_ELEMENT_COLUMN] as $key) {
+// if (isset($arrShow[$key])) {
+// if ($arrShow[$key] == '') {
+// unset($arrShow[$key]);
+// }
+// }
+// }
+
+ $arrDebugHidden[EXCEPTION_FILE] = $this->getFile();
+ $arrDebugHidden[EXCEPTION_LINE] = $this->getLine();
+
+ $arrTrace = $this->getExtensionTraceAsArray();
+ if ($store !== null) {
$this->messageArrayDebug[EXCEPTION_MESSAGE_DEBUG] = Store::getVar(EXCEPTION_MESSAGE_DEBUG, STORE_SYSTEM);
@@ -79,41 +102,66 @@ class AbstractException extends \Exception {
$arrDebugShow[EXCEPTION_PAGE_ID] = $t3Vars[TYPO3_PAGE_ID];
$arrDebugShow[EXCEPTION_TT_CONTENT_UID] = $t3Vars[TYPO3_TT_CONTENT_UID];
- $arrDebugHidden = array();
- $arrDebugHidden[EXCEPTION_FILE] = $this->getFile();
- $arrDebugHidden[EXCEPTION_LINE] = $this->getLine();
- $arrDebugHidden[EXCEPTION_STACKTRACE] = $this->getTraceAsString();
-
// Optional existing arrays will be flatten.
$arrDebugShow = OnArray::varExportArray($arrDebugShow);
$arrDebugHidden = OnArray::varExportArray($arrDebugHidden);
- // Sanitize any HTML content.
- $arrDebugShow = OnArray::htmlentitiesOnArray($arrDebugShow);
- $arrDebugHidden = OnArray::htmlentitiesOnArray($arrDebugHidden);
+ // Debug Information
+ if (Support::findInSet(SYSTEM_SHOW_DEBUG_INFO_YES, $store->getVar(SYSTEM_SHOW_DEBUG_INFO, STORE_SYSTEM))) {
- // In case there is a 'form' name given in SIP, we probably have a problem in a form and a direct link to
- // edit the broken form will be helpful.
- $storeSystem = $store->getStore(STORE_SYSTEM);
- if (!empty($storeSystem[SYSTEM_FORM])) {
- $arrDebugShow[EXCEPTION_EDIT_FORM] = $this->buildFormLink($storeSystem);
- }
+ // In case there is a 'form' name given in SIP, we probably have a problem in a form and a direct link to
+ // edit the broken form will be helpful.
+ $storeSystem = $store->getStore(STORE_SYSTEM);
+ if (!empty($storeSystem[SYSTEM_FORM])) {
+ $arrDebugShow[EXCEPTION_EDIT_FORM] = $this->buildFormLink($storeSystem);
+ }
- $htmlDebug = OnArray::arrayToHtmlTable($arrDebugShow, 'Debug', EXCEPTION_TABLE_CLASS);
- $hidden = OnArray::arrayToHtmlTable($arrDebugHidden, 'Details', EXCEPTION_TABLE_CLASS);
+ $htmlDebug = OnArray::arrayToHtmlTable(OnArray::htmlentitiesOnArray(array_merge($arrMsg, $arrDebugShow)), 'Debug', EXCEPTION_TABLE_CLASS);
- // Show / hide with just CSS: http://jsfiddle.net/t5Nf8/1/
- $htmlDebug .= "<style>input[type=checkbox]:checked + label + table { display: none; }</style>" .
- "<input type='checkbox' checked id='stacktrace'><label for='stacktrace'> Show/hide more details</label>$hidden";
+ $arrDebugHiddenClean = OnArray::htmlentitiesOnArray($arrDebugHidden);
+ $arrDebugHiddenClean[EXCEPTION_STACKTRACE] = implode($arrTrace, '<br>');
+
+ $hidden = OnArray::arrayToHtmlTable($arrDebugHiddenClean, 'Details', EXCEPTION_TABLE_CLASS);
+
+ // Show / hide with just CSS: http://jsfiddle.net/t5Nf8/1/
+ $htmlDebug .= "<style>input[type=checkbox]:checked + label + table { display: none; }</style>" .
+ "<input type='checkbox' checked id='stacktrace'><label for='stacktrace'> Show/hide more details</label>$hidden";
+ }
}
+ $qfqLog = $store->getVar(SYSTEM_QFQ_LOG, STORE_SYSTEM);
+ $arrDebugHidden[EXCEPTION_STACKTRACE] = PHP_EOL . implode($arrTrace, PHP_EOL);
+ $arrLogAll = array_merge($arrMsg, $arrShow, $arrDebugShow, $arrDebugHidden);
+ $logAll = OnArray::arrayToLog($arrLogAll);
+ Logger::logMessage($logAll, $qfqLog);
+
// Sanitize any HTML content.
$arrShow = OnArray::htmlentitiesOnArray($arrShow);
- $html = OnArray::arrayToHtmlTable($arrShow, 'Error', EXCEPTION_TABLE_CLASS);
- return $html . $htmlDebug;
+ return $this->formatMessageUser($arrShow) . $htmlDebug;
+ }
+
+ /**
+ * @return array
+ */
+ private function getExtensionTraceAsArray() {
+
+ $trace = $this->getTraceAsString();
+ $arrTrace = explode(PHP_EOL, $trace);
+
+ return OnArray::filterValueSubstring($arrTrace, '/typo3conf/ext/' . EXT_KEY . '/');
}
+ /**
+ * @param $arrShow
+ * @return string
+ */
+ private function formatMessageUser($arrShow) {
+
+ $html = '<p><em>' . $arrShow[EXCEPTION_TIMESTAMP] . ', Reference: ' . $arrShow[EXCEPTION_UNIQID] . '</em></p>';
+ $html .= '<p>' . $arrShow[EXCEPTION_MESSAGE] . '</p>';
+ return $html;
+ }
/**
* Build a FormEditor link to the broken form.
diff --git a/extension/qfq/qfq/helper/OnArray.php b/extension/qfq/qfq/helper/OnArray.php
index 1604e491b..f87ac3768 100644
--- a/extension/qfq/qfq/helper/OnArray.php
+++ b/extension/qfq/qfq/helper/OnArray.php
@@ -110,6 +110,26 @@ class OnArray {
return $result;
}
+ /**
+ * Iterates over all elements and return those with $needle in $row
+ *
+ * @param array $dataArray
+ * @param $needle
+ * @return array
+ */
+ public static function filterValueSubstring(array $dataArray, $needle) {
+ $result = array();
+
+ foreach ($dataArray as $row) {
+ if (strpos($row, $needle) !== false) {
+ $result[] = $row;
+ }
+ }
+
+ return $result;
+ }
+
+
/**
* Converts a onedimensional array by using htmlentities on all elements
*
@@ -134,7 +154,7 @@ class OnArray {
public static function varExportArray(array $arr) {
foreach ($arr as $key => $value) {
if (is_array($value)) {
- $arr[$key] = var_export($value, true);
+ $arr[$key] = empty($value) ? 'array ()' : var_export($value, true);
}
}
@@ -159,6 +179,27 @@ class OnArray {
return "<table class='$class'><thead><tr><th colspan='2'>$title</th></tr></thead>$html</table>";
}
+ /**
+ * @param array $arr
+ * @return string
+ */
+ public static function arrayToLog(array $arr) {
+
+ $output = EXCEPTION_UNIQID . ':: ';
+ $output .= (empty($arr[EXCEPTION_UNIQID])) ? ' - ' : $arr[EXCEPTION_UNIQID];
+ $output .= PHP_EOL . '------------------------------------------------' . PHP_EOL;
+
+ foreach ($arr as $key => $value) {
+ if (!empty($value) && $key != EXCEPTION_UNIQID) {
+ $output .= $key . ':: ' . $value . PHP_EOL;
+ }
+ }
+
+ $output .= '==================================================' . PHP_EOL;
+
+ return $output;
+ }
+
/**
* Split Array around $str to $arr around $delimiter. Escaped $delimiter will be preserved.
*
@@ -387,5 +428,4 @@ class OnArray {
public static function getMd5(array $data) {
return md5(implode($data));
}
-
}
\ No newline at end of file
diff --git a/extension/qfq/qfq/store/Config.php b/extension/qfq/qfq/store/Config.php
index ac41d1fd0..8677af4d9 100644
--- a/extension/qfq/qfq/store/Config.php
+++ b/extension/qfq/qfq/store/Config.php
@@ -257,6 +257,7 @@ class Config {
SYSTEM_DATE_FORMAT => 'yyyy-mm-dd',
SYSTEM_SHOW_DEBUG_INFO => SYSTEM_SHOW_DEBUG_INFO_AUTO,
+ SYSTEM_QFQ_LOG => SYSTEM_QFQ_LOG_FILE,
SYSTEM_SQL_LOG => SYSTEM_SQL_LOG_FILE,
SYSTEM_SQL_LOG_MODE => 'modify',
SYSTEM_MAIL_LOG => SYSTEM_MAIL_LOG_FILE,
--
GitLab