Merge branch 'F4434DisableSpecialColumnNamesStartingWithoutUnderscore' into 'master'

fixes #4434 Disable special column names starting without underscore

See merge request !174

extension/Classes/Core/Constants.php

@@ -1547,6 +1547,13 @@ const COLUMN_EXCEL_NUMERIC = 'XLSn';
 const COLUMN_EXCEL_STRING = 'XLSs';
 const COLUMN_EXCEL_BASE64 = 'XLSb';
 
+const COLUMN_BULLET = "bullet";
+const COLUMN_CHECK = "check";
+const COLUMN_IMG = "img";
+const COLUMN_MAILTO = "mailto";
+const COLUMN_SENDMAIL = "sendmail";
+const COLUMN_VERTICAL = "vertical";
+
 const COLUMN_WRAP_TOKEN = '+';
 const COLUMN_STORE_USER = '=';
 
@@ -1893,6 +1900,11 @@ const QFQ_VERSION_KEY_FUNCTION_VERSION = 'functionVersion';
 const QFQ_VERSION_KEY = 'Version';
 const QFQ_FUNCTION_SQL = 'function.sql';
 
+// update special column names (add '_' in front)
+const ACTION_SPECIAL_COLUMN_UPDATE = '_scupdate';  // get parameter to set the update behaviour
+const ACTION_SPECIAL_COLUMN_DO_REPLACE = 'replace'; // special columns are automatically replaced
+const ACTION_SPECIAL_COLUMN_DO_SKIP_REPLACE = 'skip_replace'; // special columns are automatically replaced
+
 // tablesorter
 const TABLESORTER_VIEW_SAVER = 'tablesorter-view-saver';
 const DATA_TABLESORTER_VIEW = 'data-tablesorter-view';

extension/Classes/Core/Database/DatabaseUpdate.php

@@ -9,6 +9,7 @@
 namespace IMATHUZH\Qfq\Core\Database;
 
 use IMATHUZH\Qfq\Core\Helper\Logger;
+use IMATHUZH\Qfq\Core\Store\Store;
 
 
 /*
@@ -32,14 +33,18 @@ class DatabaseUpdate {
 
     /**
      * @var Database instantiated class
+     * @var Store instantiated class
      */
     protected $db = null;
+    protected $store = null;
 
     /**
      * @param Database $db
+     * @param Store $store
      */
-    public function __construct(Database $db) {
+    public function __construct(Database $db, Store $store) {
         $this->db = $db;
+        $this->store = $store;
     }
 
     /**
@@ -129,6 +134,10 @@ class DatabaseUpdate {
         $versionInfo = $this->getDatabaseVersion();
         $old = $versionInfo[QFQ_VERSION_KEY] ?? false;
 
+        if (version_compare($old, '19.9.9') === -1) {  #TODO: change version number before new release! (replace 19.9.9 with the version of the new release)
+            $this->updateSpecialColumns();
+        }
+
         if ($dbUpdate === SYSTEM_DB_UPDATE_ALWAYS || ($dbUpdate === SYSTEM_DB_UPDATE_AUTO && $new != $old)) {
 
             $newFunctionHash = $this->updateSqlFunctions($versionInfo[QFQ_VERSION_KEY_FUNCTION_HASH] ?? '');
@@ -154,12 +163,120 @@ class DatabaseUpdate {
         }
     }
 
+    /**
+     * Check if there are special columns without prepended underscore in the QFQ application. If yes, then throw an error.
+     * A link is provided to automatically prepend all found special columns. And another link to skip the auto-replacement.
+     *
+     * @throws \CodeException
+     * @throws \DbException
+     * @throws \UserFormException
+     */
+    private function updateSpecialColumns() {
+        // Prepare regex patterns to find "AS <special column name>"
+        $special_columns = ['link', 'exec', 'Page', 'Pagec', 'Paged', 'Pagee', 'Pageh', 'Pagei', 'Pagen', 'Pages'
+            , 'page', 'pagec', 'paged', 'pagee', 'pageh', 'pagei', 'pagen', 'pages', 'yank', 'Pdf', 'File', 'Zip'
+            , 'pdf', 'file', 'zip', 'excel', 'savePdf', 'thumbnail', 'monitor', 'mimeType', 'fileSize', 'nl2br'
+            , 'htmlentities', 'striptags', 'XLS', 'XLSs', 'XLSb', 'XLSn', 'bullet', 'check', 'img', 'mailto'
+            , 'sendmail', 'vertical'];
+        $make_pattern = function ($column) {
+            return '/([aA][sS]\s+)(' . $column . ')/s';
+        };
+        $patterns = array_map($make_pattern, $special_columns);
+
+        // Prepare search and replace
+        $placeholder = '%%%UNDERLINE%%%';  // used temporarily to mark where '_' should go
+        $actionSpecialColumn = $_GET[ACTION_SPECIAL_COLUMN_UPDATE] ?? ''; // get parameter to decide whether to execute the replacement
+        $dbT3 = $this->store->getVar(SYSTEM_DB_NAME_T3, STORE_SYSTEM);
+        $message = ''; // error message in case an old special column is found
+
+        // TT_CONTENT tt_content.bodytext
+        $message_fe = '';
+        if (defined('PHPUNIT_QFQ')) {
+            $res = array();
+        } else {
+            $res = $this->db->sql("SELECT uid, header, bodytext FROM " . $dbT3 . ".tt_content WHERE CType='qfq_qfq' AND deleted=0;");
+        }
+        foreach ($res as $i => $tt_content) {
+            $replaced_placeholder = preg_replace($patterns, '${1}' . $placeholder . '${2}', $tt_content['bodytext']);
+            if (strpos($replaced_placeholder, $placeholder) !== false) {
+                if ($actionSpecialColumn === ACTION_SPECIAL_COLUMN_DO_REPLACE) {
+                    $replace = str_replace($placeholder, '_', $replaced_placeholder);
+                    $query = "UPDATE " . $dbT3 . ".tt_content SET bodytext='" . addslashes($replace) . "' WHERE uid='" . $tt_content['uid'] . "'";
+                    $this->db->sql($query);
+                }
+                $message_fe .= '<hr><b>' . $tt_content['header'] . ' [uid:' . $tt_content['uid'] . ']</b><br><br>';
+                $message_fe .= str_replace($placeholder,
+                    '<span style="font-weight: bold; color: red;">>>>_</span>',
+                    htmlentities($replaced_placeholder));
+            }
+        }
+        if ($message_fe != '') {
+            $message .= '<hr><h3>Typo3 Table: tt_content (column: bodytext)</h3>' . $message_fe;
+        }
+
+        // FORM ELEMENTS FormElement.value, FormElement.note
+        $message_ttc = '';
+        if (defined('PHPUNIT_QFQ')) {
+            $res = array();
+        } else {
+            $res = $this->db->sql("SELECT fe.id, fe.name, fe.value, fe.note FROM FormElement as fe WHERE fe.type='note' AND fe.value LIKE '#!report%' OR fe.note LIKE '%#!report%';");
+        }
+        foreach ($res as $i => $tt_content) {
+
+            foreach (['value', 'note'] as $j => $columnName) {
+                $replaced_placeholder = preg_replace($patterns, '${1}' . $placeholder . '${2}', $tt_content[$columnName]);
+                if (strpos($replaced_placeholder, $placeholder) !== false) {
+                    if ($actionSpecialColumn === ACTION_SPECIAL_COLUMN_DO_REPLACE) {
+                        $replace = str_replace($placeholder, '_', $replaced_placeholder);
+                        $query = "UPDATE FormElement SET " . $columnName . "='" . addslashes($replace) . "' WHERE id='" . $tt_content['id'] . "'";
+                        $this->db->sql($query);
+                    }
+                    $message_ttc .= '<hr><b>' . $tt_content['name'] . ' [id:' . $tt_content['id'] . '] (FormElement.' . $columnName . ')</b><br><br>';
+                    $message_ttc .= str_replace($placeholder,
+                        '<span style="font-weight: bold; color: red;">>>>_</span>',
+                        htmlentities($replaced_placeholder));
+                }
+            }
+        }
+        if ($message_ttc != '') {
+            $message .= '<hr><h3>QFQ Table: FormElement (columns: value and note)</h3>' . $message_ttc;
+        }
+
+        // show error message or save log
+        if ($message != '') {
+            if ($actionSpecialColumn === ACTION_SPECIAL_COLUMN_DO_REPLACE) {
+                // save log file
+                $message = '<h1>Special column names replaced</h1>The following special column names were replaced.<hr>' . $message;
+                Logger::logMessage($message, SYSTEM_FILEADMIN_PROTECTED_LOG . '/' . date("YmdHi") . '_special_columns_auto_update.html');
+            } elseif ($actionSpecialColumn === ACTION_SPECIAL_COLUMN_DO_SKIP_REPLACE) {
+                // do nothing
+            } else {
+                // show error
+                $message = $actionSpecialColumn
+                    . '<h2>Special Column names without prepended underscore found.</h2>'
+                    . ' Those are not supported any longer.'
+                    . '<h2>SOLUTION</h2>'
+                    . 'Click <a href="?' . http_build_query(array_merge($_GET, array(ACTION_SPECIAL_COLUMN_UPDATE => ACTION_SPECIAL_COLUMN_DO_REPLACE))) . '">Auto-Replace</a>'
+                    . ' to automatically prepend the found column names with an underscore.'
+                    . ' In the report below the missing underscores are marked by "<span style="font-weight: bold; color: red;">>>>_</span>".'
+                    . ' This report will be saved in ' . SYSTEM_FILEADMIN_PROTECTED_LOG . ' after the automatic replacement.'
+                    . ' <br><br>To update qfq without changing the special columns (your app will probably be broken): '
+                    . '<a href="?' . http_build_query(array_merge($_GET, array(ACTION_SPECIAL_COLUMN_UPDATE => ACTION_SPECIAL_COLUMN_DO_SKIP_REPLACE))) . '">Skip Auto-Replace</a>'
+                    . '<h2>Report</h2>'
+                    . $message;
+                $errorMsg[ERROR_MESSAGE_TO_USER] = 'Error while updating qfq. ';
+                $errorMsg[ERROR_MESSAGE_TO_DEVELOPER] = $message;
+                $errorMsg[ERROR_MESSAGE_TO_DEVELOPER_SANITIZE] = false;
+                throw new \DbException(json_encode($errorMsg), ERROR_PLAY_SQL_FILE);
+            }
+        }
+    }
+
     /**
      * @param $oldFunctionsHash
      *
      * @return string
      *
-     * @throws \CodeException
      * @throws \DbException
      * @throws \UserFormException
      */

extension/Classes/Core/QuickFormQuery.php

@@ -173,7 +173,7 @@ class QuickFormQuery {
         $this->evaluate = new Evaluate($this->store, $this->dbArray[$this->dbIndexData]);
 
         $dbUpdate = $this->store->getVar(SYSTEM_DB_UPDATE, STORE_SYSTEM);
-        $updateDb = new DatabaseUpdate($this->dbArray[$this->dbIndexQfq]);
+        $updateDb = new DatabaseUpdate($this->dbArray[$this->dbIndexQfq], $this->store);
         $updateDb->checkNupdate($dbUpdate);
 
         $this->store->FillStoreSystemBySql(); // Do this after the DB-update

extension/Tests/Unit/Core/Database/DatabaseUpdateTest.php

@@ -6,7 +6,8 @@
 namespace IMATHUZH\Qfq\Tests\Unit\Core\Database;
 
 use IMATHUZH\Qfq\Core\Database\DatabaseUpdate;
- 
+use IMATHUZH\Qfq\Core\Store\Store;
+
 
 require_once(__DIR__ . '/AbstractDatabaseTest.php');
 
@@ -29,13 +30,16 @@ class DatabaseUpdateTest extends AbstractDatabaseTest {
      * @throws \CodeException
      * @throws \DbException
      * @throws \UserFormException
+     * @throws \UserReportException
      */
     public function testCheckNupdate() {
 
 //        $countQfqTables = 9;
         $countQfqTables = 10;
 
-        $dbUpdate = new DatabaseUpdate($this->dbArray[DB_INDEX_DEFAULT]);
+        $store = Store::getInstance();
+
+        $dbUpdate = new DatabaseUpdate($this->dbArray[DB_INDEX_DEFAULT], $store);
 
         $dbUpdate->checkNupdate(SYSTEM_DB_UPDATE_NEVER);
         $allRows = $this->dbArray[DB_INDEX_DEFAULT]->sql('SHOW tables;');