F5458 Error Log Unique Identifier: Exception message might be an ordinary...

F5458 Error Log Unique Identifier: Exception message might be an ordinary string or a JSON encoded array.

doc/NewVersion.md

@@ -62,4 +62,9 @@ https://docs.typo3.org/typo3cms/drafts/github/T3DocumentationStarter/Public-Info
 
     make update-qfq-doc
 
+Upload new version to TER
+=========================
 
+* https://extensions.typo3.org/ > Log in > My Extensions.
+* Rename the ZIP file to be TER compatible: e.g. qfq_18.6.0.zip.
+* Upload - that's all.
\ No newline at end of file

extension/Documentation/Manual.rst

@@ -262,11 +262,15 @@ Setup a *report* to manage all *forms*:
 Install Check List
 ------------------
 
-* Protect the directory `<T3 installation>/fileadmin/protected` in Apache against direct file access. Those directory
-  should be used for confidential (uploaded / generated) data.
+* Protect the directory `<T3 installation>/fileadmin/protected` in Apache against direct file access.
+
+  * `<T3 installation>/fileadmin/protected/` should be used for confidential (uploaded / generated) data.
+  * `<T3 installation>/fileadmin/protected/log/...` is the default place for QFQ logfiles.
+
 * Protect the directory `<T3 installation>/fileadmin` in Apache to not execute PHP Scripts - malicious uploads won't be executed.
 * Setup a log rotation rule for `sqlLog`.
-* Check that `sqlLogMode` is set to `modify`  on productive sites.
+* Check that `sqlLogMode` is set to `modify` on productive sites. With `none` you have no chance to find out who changed
+  which data and `all` really logs a mass of data.
 
 .. _configuration:
 

extension/ext_conf_template.txt

@@ -48,6 +48,9 @@ fillStoreSystemBySqlErrorMsg3 =
 
 
 
+# cat=debug/qfq; type=string; label=QFQ log file:Default is 'fileadmin/protected/log/qfq.log'. A logfile of fired SQL statements. PathFile is absolute or relative to '<site path>'.
+qfqLog = fileadmin/protected/log/qfq.log
+
 # cat=debug/sql; type=string; label=SQL log mode:Default is 'modify'. A logfile of QFQ fired SQL statements will be written. Possible modes are 'all' - every statement will be logged (this might a lot). 'modify' - log only statements who change data. 'error' - log only DB errors. 'none' - log never.
 sqlLogMode = modify
 

extension/qfq/qfq/Constants.php

@@ -111,6 +111,10 @@ const KVP_IF_VALUE_EMPTY_COPY_KEY = 'if_value_empty_copy_key';
 const KVP_VALUE_GIVEN = 'value_given';
 
 // https://lib2.colostate.edu/wildlife/atoz.php?letter=ALL
+// JSON encoded messages thrown through an exception:
+const ERROR_MESSAGE_TO_USER = 'toUser'; // always shown to the user.
+const ERROR_MESSAGE_SUPPORT = 'support'; // Message to help the developer to understand the problem.
+const ERROR_MESSAGE_OS = 'os'; // Error message from the OS - like 'file not found' or specific SQL problem
 
 // QFQ Error Codes
 const ERROR_UNKNOW_SANITIZE_CLASS = 1001;
@@ -174,7 +178,7 @@ const ERROR_MISSING_OPEN_DELIMITER = 1060;
 const ERROR_MISSING_CLOSE_DELIMITER = 1061;
 const ERROR_EXPECTED_ARRAY = 1062;
 const ERROR_REPORT_FAILED_ACTION = 1063;
-const ERROR_MISSING_MESSAGE_FAIL = 1064;
+
 const ERROR_MISSING_TABLE_NAME = 1065;
 const ERROR_MISSING_TABLE = 1066;
 const ERROR_RECORD_NOT_FOUND = 1067;
@@ -403,13 +407,17 @@ const SYSTEM_DB_INDEX_QFQ = "indexQfq";
 const SYSTEM_DB_NAME_DATA = '_dbNameData';
 const SYSTEM_DB_NAME_QFQ = '_dbNameQfq';
 
+const SYSTEM_QFQ_LOG = 'qfqLog'; //  Logging to file
+const SYSTEM_QFQ_LOG_FILE = 'fileadmin/protected/log/qfq.log';
+
 const SYSTEM_SQL_LOG = 'sqlLog'; //  Logging to file
-const SYSTEM_SQL_LOG_FILE = '../../sql.log';
+const SYSTEM_SQL_LOG_FILE = 'fileadmin/protected/log/sql.log';
 const SYSTEM_SQL_LOG_MODE = 'sqlLogMode'; // Mode, which statements to log.
+
 const SYSTEM_DATE_FORMAT = 'dateFormat';
 const SYSTEM_REDIRECT_ALL_MAIL_TO = 'redirectAllMailTo';
 const SYSTEM_MAIL_LOG = 'mailLog';
-const SYSTEM_MAIL_LOG_FILE = '../../mail.log';
+const SYSTEM_MAIL_LOG_FILE = 'fileadmin/protected/log/mail.log';
 
 const SYSTEM_SHOW_DEBUG_INFO = 'showDebugInfo';
 const SYSTEM_SHOW_DEBUG_INFO_YES = 'yes';
@@ -1452,6 +1460,7 @@ const EXCEPTION_TT_CONTENT_UID = 'Content Id';
 const EXCEPTION_EDIT_FORM = 'Edit';
 
 const EXCEPTION_TIMESTAMP = 'Timestamp';
+const EXCEPTION_UNIQID = 'UniqId';
 const EXCEPTION_CODE = 'Code';
 const EXCEPTION_MESSAGE = 'Message'; // Will be shown on every exception. NO sensitive data here!
 const EXCEPTION_MESSAGE_DEBUG = SYSTEM_MESSAGE_DEBUG;  // Will only be shown as debugging (Typically BE user is logged in)

extension/qfq/qfq/database/Database.php

@@ -67,6 +67,7 @@ class Database {
      * @throws CodeException
      * @throws DbException
      * @throws UserFormException
+     * @throws UserReportException
      */
     public function __construct($dbIndex = DB_INDEX_DEFAULT) {
         if (empty($dbIndex)) {
@@ -275,6 +276,25 @@ class Database {
         $this->mysqli_result = null;
     }
 
+    /**
+     * Checks the problematic $sql if there is a common mistake.
+     * If something is found, give a hint.
+     *
+     * @param $sql
+     * @return string
+     */
+    private function getSqlHint($sql){
+
+        // Check if there is a comma before FROM: 'SELECT ... , FROM ...'
+        $pos=stripos($sql, ' FROM ');
+        if($pos!==false && $pos>0 && $sql[$pos -1]==','){
+            return 'HINT: remove extra "," before FROM';
+        }
+
+
+        return '';
+    }
+
     /**
      * Execute a prepared SQL statement like SELECT, INSERT, UPDATE, DELETE, SHOW, ...
      *
@@ -298,32 +318,43 @@ class Database {
         $sqlLogMode = $this->isSqlModify($sql) ? SQL_LOG_MODE_MODIFY : SQL_LOG_MODE_ALL;
         $result = 0;
         $stat = array();
+        $errorMsg[ERROR_MESSAGE_TO_USER] = empty($specificMessage) ? 'SQL error' : $specificMessage;
 
         if ($this->store !== null) {
             $this->store->setVar(SYSTEM_SQL_FINAL, $sql, STORE_SYSTEM);
             $this->store->setVar(SYSTEM_SQL_PARAM_ARRAY, $parameterArray, STORE_SYSTEM);
         }
-        if ($specificMessage !== '') {
-            $specificMessage = ' - ' . $specificMessage;
-        }
+
+//        if ($specificMessage !== '') {
+//            $specificMessage = ' - ' . $specificMessage;
+//        }
         // Logfile
         $this->dbLog($sqlLogMode, $sql, $parameterArray);
 
         if (false === ($this->mysqli_stmt = $this->mysqli->prepare($sql))) {
             $this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
-            throw new DbException('[ mysqli: ' . $this->mysqli->errno . ' ] ' . $this->mysqli->error . $specificMessage, ERROR_DB_PREPARE);
+            $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+            $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' . $this->mysqli->errno . ' ] ' . $this->mysqli->error;
+
+            throw new DbException(json_encode($errorMsg), ERROR_DB_PREPARE);
         }
 
         if (count($parameterArray) > 0) {
             if (false === $this->prepareBindParam($parameterArray)) {
                 $this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
-                throw new DbException('[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error . $specificMessage, ERROR_DB_BIND);
+                $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+                $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error;
+
+                throw new DbException(json_encode($errorMsg), ERROR_DB_BIND);
             }
         }
 
         if (false === $this->mysqli_stmt->execute()) {
             $this->dbLog(SQL_LOG_MODE_ERROR, $sql, $parameterArray);
-            throw new DbException('[ mysqli: ' . $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error . $specificMessage, ERROR_DB_EXECUTE);
+            $errorMsg[ERROR_MESSAGE_SUPPORT] = $this->getSqlHint($sql);
+            $errorMsg[ERROR_MESSAGE_OS] = '[ mysqli: ' .  $this->mysqli_stmt->errno . ' ] ' . $this->mysqli_stmt->error;
+
+            throw new DbException(json_encode($errorMsg),ERROR_DB_EXECUTE);
         }
 
         $msg = '';
@@ -595,6 +626,9 @@ class Database {
      * @param string $columnName name of the column
      *
      * @return array
+     * @throws CodeException
+     * @throws DbException
+     * @throws UserFormException
      */
     public function getEnumSetValueList($table, $columnName) {
 

extension/qfq/qfq/exceptions/AbstractException.php

@@ -16,6 +16,7 @@ require_once(__DIR__ . '/../report/Link.php');
 require_once(__DIR__ . '/../database/Database.php');
 require_once(__DIR__ . '/UserFormException.php');
 require_once(__DIR__ . '/../helper/OnArray.php');
+require_once(__DIR__ . '/../helper/Logger.php');
 
 
 /**
@@ -33,14 +34,22 @@ class AbstractException extends \Exception {
     protected $line = '';
 
     /**
+     * There are 3+1 different messages:
+     * 'toUser' - shown in the client to the user - no details here!!!
+     * 'support' - help for the developer
+     * 'os' - message from the OS, like 'file not found'
+     * Stacktrace, Form, FormElement, Report level, T3 page, T3 tt_content uid, ...
+     *
      * @return string
      * @throws CodeException
      * @throws UserFormException
      */
     public function formatException() {
+
         $t3Vars = array();
         $arrShow = $this->messageArray;
         $htmlDebug = '';
+        $arrDebugShow = array();
 
         try {
             // In a very early stage, it might be possible that Store can't be initialized: take care not to use it.
@@ -48,7 +57,7 @@ class AbstractException extends \Exception {
             $t3Vars = $store->getStore(STORE_TYPO3);
         } catch (CodeException $e) {
             $store = null;
-        } catch (\Exception $exception){
+        } catch (\Exception $exception) {
             $store = null;
         }
 
@@ -58,18 +67,32 @@ class AbstractException extends \Exception {
 
         $arrShow[EXCEPTION_TIMESTAMP] = date('Y.m.d H:i:s O');
         $arrShow[EXCEPTION_CODE] = $this->getCode();
-        $arrShow[EXCEPTION_MESSAGE] = $this->getMessage();
-
-        foreach ([EXCEPTION_FORM, EXCEPTION_FORM_ELEMENT, EXCEPTION_FORM_ELEMENT_COLUMN] as $key) {
-            if (isset($arrShow[$key])) {
-                if ($arrShow[$key] == '') {
-                    unset($arrShow[$key]);
-                }
-            }
+        $arrShow[EXCEPTION_UNIQID] = uniqid();
+
+        // Get exception message and if JSON, decode it.
+        $msg = $this->getMessage();
+        $arrMsg = json_decode($msg, true);
+        if ($arrMsg === null) {
+            $arrShow[EXCEPTION_MESSAGE] = $msg;
+            $arrMsg[ERROR_MESSAGE_TO_USER] = $msg;
+        } else {
+            $arrShow[EXCEPTION_MESSAGE] = $arrMsg[ERROR_MESSAGE_TO_USER];
         }
 
-        // Debug Information
-        if ($store !== null && Support::findInSet(SYSTEM_SHOW_DEBUG_INFO_YES, $store->getVar(SYSTEM_SHOW_DEBUG_INFO, STORE_SYSTEM))) {
+//        // Unset empty elements
+//        foreach ([EXCEPTION_FORM, EXCEPTION_FORM_ELEMENT, EXCEPTION_FORM_ELEMENT_COLUMN] as $key) {
+//            if (isset($arrShow[$key])) {
+//                if ($arrShow[$key] == '') {
+//                    unset($arrShow[$key]);
+//                }
+//            }
+//        }
+
+        $arrDebugHidden[EXCEPTION_FILE] = $this->getFile();
+        $arrDebugHidden[EXCEPTION_LINE] = $this->getLine();
+
+        $arrTrace = $this->getExtensionTraceAsArray();
+        if ($store !== null) {
 
             $this->messageArrayDebug[EXCEPTION_MESSAGE_DEBUG] = Store::getVar(EXCEPTION_MESSAGE_DEBUG, STORE_SYSTEM);
 
@@ -79,41 +102,66 @@ class AbstractException extends \Exception {
             $arrDebugShow[EXCEPTION_PAGE_ID] = $t3Vars[TYPO3_PAGE_ID];
             $arrDebugShow[EXCEPTION_TT_CONTENT_UID] = $t3Vars[TYPO3_TT_CONTENT_UID];
 
-            $arrDebugHidden = array();
-            $arrDebugHidden[EXCEPTION_FILE] = $this->getFile();
-            $arrDebugHidden[EXCEPTION_LINE] = $this->getLine();
-            $arrDebugHidden[EXCEPTION_STACKTRACE] = $this->getTraceAsString();
-
             // Optional existing arrays will be flatten.
             $arrDebugShow = OnArray::varExportArray($arrDebugShow);
             $arrDebugHidden = OnArray::varExportArray($arrDebugHidden);
 
-            // Sanitize any HTML content.
-            $arrDebugShow = OnArray::htmlentitiesOnArray($arrDebugShow);
-            $arrDebugHidden = OnArray::htmlentitiesOnArray($arrDebugHidden);
+            // Debug Information
+            if (Support::findInSet(SYSTEM_SHOW_DEBUG_INFO_YES, $store->getVar(SYSTEM_SHOW_DEBUG_INFO, STORE_SYSTEM))) {
 
-            // In case there is a 'form' name given in SIP, we probably have a problem in a form and a direct link to
-            // edit the broken form will be helpful.
-            $storeSystem = $store->getStore(STORE_SYSTEM);
-            if (!empty($storeSystem[SYSTEM_FORM])) {
-                $arrDebugShow[EXCEPTION_EDIT_FORM] = $this->buildFormLink($storeSystem);
-            }
+                // In case there is a 'form' name given in SIP, we probably have a problem in a form and a direct link to
+                // edit the broken form will be helpful.
+                $storeSystem = $store->getStore(STORE_SYSTEM);
+                if (!empty($storeSystem[SYSTEM_FORM])) {
+                    $arrDebugShow[EXCEPTION_EDIT_FORM] = $this->buildFormLink($storeSystem);
+                }
 
-            $htmlDebug = OnArray::arrayToHtmlTable($arrDebugShow, 'Debug', EXCEPTION_TABLE_CLASS);
-            $hidden = OnArray::arrayToHtmlTable($arrDebugHidden, 'Details', EXCEPTION_TABLE_CLASS);
+                $htmlDebug = OnArray::arrayToHtmlTable(OnArray::htmlentitiesOnArray(array_merge($arrMsg, $arrDebugShow)), 'Debug', EXCEPTION_TABLE_CLASS);
 
-            // Show / hide with just CSS: http://jsfiddle.net/t5Nf8/1/
-            $htmlDebug .= "<style>input[type=checkbox]:checked + label + table { display: none; }</style>" .
-                "<input type='checkbox' checked id='stacktrace'><label for='stacktrace'>&nbsp;Show/hide more details</label>$hidden";
+                $arrDebugHiddenClean = OnArray::htmlentitiesOnArray($arrDebugHidden);
+                $arrDebugHiddenClean[EXCEPTION_STACKTRACE] = implode($arrTrace, '<br>');
+
+                $hidden = OnArray::arrayToHtmlTable($arrDebugHiddenClean, 'Details', EXCEPTION_TABLE_CLASS);
+
+                // Show / hide with just CSS: http://jsfiddle.net/t5Nf8/1/
+                $htmlDebug .= "<style>input[type=checkbox]:checked + label + table { display: none; }</style>" .
+                    "<input type='checkbox' checked id='stacktrace'><label for='stacktrace'>&nbsp;Show/hide more details</label>$hidden";
+            }
         }
 
+        $qfqLog = $store->getVar(SYSTEM_QFQ_LOG, STORE_SYSTEM);
+        $arrDebugHidden[EXCEPTION_STACKTRACE] = PHP_EOL . implode($arrTrace, PHP_EOL);
+        $arrLogAll = array_merge($arrMsg, $arrShow, $arrDebugShow, $arrDebugHidden);
+        $logAll = OnArray::arrayToLog($arrLogAll);
+        Logger::logMessage($logAll, $qfqLog);
+
         // Sanitize any HTML content.
         $arrShow = OnArray::htmlentitiesOnArray($arrShow);
-        $html = OnArray::arrayToHtmlTable($arrShow, 'Error', EXCEPTION_TABLE_CLASS);
 
-        return $html . $htmlDebug;
+        return $this->formatMessageUser($arrShow) . $htmlDebug;
+    }
+
+    /**
+     * @return array
+     */
+    private function getExtensionTraceAsArray() {
+
+        $trace = $this->getTraceAsString();
+        $arrTrace = explode(PHP_EOL, $trace);
+
+        return OnArray::filterValueSubstring($arrTrace, '/typo3conf/ext/' . EXT_KEY . '/');
     }
+    /**
+     * @param $arrShow
+     * @return string
+     */
+    private function formatMessageUser($arrShow) {
+
+        $html = '<p><em>' . $arrShow[EXCEPTION_TIMESTAMP] . ', Reference: ' . $arrShow[EXCEPTION_UNIQID] . '</em></p>';
+        $html .= '<p>' . $arrShow[EXCEPTION_MESSAGE] . '</p>';
 
+        return $html;
+    }
 
     /**
      * Build a FormEditor link to the broken form.

extension/qfq/qfq/helper/OnArray.php

@@ -110,6 +110,26 @@ class OnArray {
         return $result;
     }
 
+    /**
+     * Iterates over all elements and return those with $needle in $row
+     *
+     * @param array $dataArray
+     * @param $needle
+     * @return array
+     */
+    public static function filterValueSubstring(array $dataArray, $needle) {
+        $result = array();
+
+        foreach ($dataArray as $row) {
+            if (strpos($row, $needle) !== false) {
+                $result[] = $row;
+            }
+        }
+
+        return $result;
+    }
+
+
     /**
      * Converts a onedimensional array by using htmlentities on all elements
      *
@@ -134,7 +154,7 @@ class OnArray {
     public static function varExportArray(array $arr) {
         foreach ($arr as $key => $value) {
             if (is_array($value)) {
-                $arr[$key] = var_export($value, true);
+                $arr[$key] = empty($value) ? 'array ()' : var_export($value, true);
             }
         }
 
@@ -159,6 +179,27 @@ class OnArray {
         return "<table class='$class'><thead><tr><th colspan='2'>$title</th></tr></thead>$html</table>";
     }
 
+    /**
+     * @param array $arr
+     * @return string
+     */
+    public static function arrayToLog(array $arr) {
+
+        $output = EXCEPTION_UNIQID . ':: ';
+        $output .= (empty($arr[EXCEPTION_UNIQID])) ? ' - ' : $arr[EXCEPTION_UNIQID];
+        $output .= PHP_EOL . '------------------------------------------------' . PHP_EOL;
+
+        foreach ($arr as $key => $value) {
+            if (!empty($value) && $key != EXCEPTION_UNIQID) {
+                $output .= $key . ':: ' . $value . PHP_EOL;
+            }
+        }
+
+        $output .= '==================================================' . PHP_EOL;
+
+        return $output;
+    }
+
     /**
      * Split Array around $str to $arr around $delimiter. Escaped $delimiter will be preserved.
      *
@@ -387,5 +428,4 @@ class OnArray {
     public static function getMd5(array $data) {
         return md5(implode($data));
     }
-
 }
\ No newline at end of file

extension/qfq/qfq/store/Config.php

@@ -257,6 +257,7 @@ class Config {
 
             SYSTEM_DATE_FORMAT => 'yyyy-mm-dd',
             SYSTEM_SHOW_DEBUG_INFO => SYSTEM_SHOW_DEBUG_INFO_AUTO,
+            SYSTEM_QFQ_LOG => SYSTEM_QFQ_LOG_FILE,
             SYSTEM_SQL_LOG => SYSTEM_SQL_LOG_FILE,
             SYSTEM_SQL_LOG_MODE => 'modify',
             SYSTEM_MAIL_LOG => SYSTEM_MAIL_LOG_FILE,