Merge branch 'master' into 6249-subrecord-dnd

# Conflicts:
#	extension/Documentation/Manual.rst
#	extension/qfq/qfq/Constants.php
#	extension/qfq/qfq/QuickFormQuery.php
#	extension/qfq/sql/formEditor.sql

doc/NewVersion.md

@@ -43,12 +43,12 @@ Neue Versionsnummer
    * Update the version number in this document (topic 6)
    * Commit & Push new version changes to master branch: 
    
-      New version x.y.z.
+      New version 18.8.0
 
 6) **New Tag**: 
 
-   git tag v18.6.1
-   git push -u origin v18.6.1
+   git tag v18.8.0
+   git push -u origin v18.8.0
 
 7) PhpStorm: **Sync** all files to VM qfq.
 

extension/Documentation/Release.rst

@@ -33,9 +33,45 @@ Notes
 Features
 ^^^^^^^^
 
+Version 18.8.0
+--------------
+
+Date: 26.08.2018
+
+Notes
+^^^^^
+
+* Excel Export
+* CopyCat
+
+Features
+^^^^^^^^
+
+* #4922 / Excel Export - create Excel sheets from scratch or based on a template.
+* #3294 / Improve Typo3 QFQ backend layout. Add sparql syntax highlighting.
+* #5878 / Formelement.type=note with #!report - whitespace is trimmed.
+* Import/Merge form: A new form 'copyFormFromExt' (see file `copyFormFromExt.sql`) offers a one click import of external
+  QFQ forms (incl. renumbering of id's).
+* formEditor.sql: resized Form.title from 255 to 511 (requested by IK Tool)
+* Drag and Drop now offers the possibility to show the renumbered values.
+* Manual.rst: security hints, T3 Setup best practice, text input retype, charactercountwrap.
+* Config.qfq: central defaults for DATA_MATCH, DATA_ERROR
+* Bootstrap QFQ development: switched from bower to npm only.
+
+
+    F6314: HTML Mails enabled by specifying flag 'mode=html'.
+
 Bug Fixes
 ^^^^^^^^^
 
+* #5843 / File upload: limitation to file extensions are no case insensitive.
+* #6247 / Replace deprecated each function
+* #6281 / FormElement / column 'note': token '#!report' - STORE_RECORD does not work.
+* #6331 / File Upload: Wrong error message if filesize is much too big.
+* #6229 / Add QFQ icon to content element and content element wizard
+* AbstractException.php: fixed problem with htmlEntities() on link to 'Edit Form' and 'Edit FormElement'.
+
+
 Version 18.6.1
 --------------
 

extension/Documentation/Settings.cfg

@@ -2,8 +2,8 @@
 [general]
 
 project     = QFQ - Quick Form Query
-version     = 18.6
-release     = 18.6.1
+version     = 18.8
+release     = 18.8.0
 t3author    = Carsten Rose
 copyright   = since 2017 by the author
 

extension/Documentation/_make/conf.py

@@ -57,9 +57,9 @@ copyright = u'2017, Carsten Rose'
 # built documents.lease
 #
 # The short X.Y version.
-version = '18.6'
+version = '18.8'
 # The full version, including alpha/beta/rc tags.
-release = '18.6.1'
+release = '18.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

extension/ext_emconf.php

@@ -11,7 +11,7 @@ $EM_CONF[$_EXTKEY] = array(
     'dependencies' => 'fluid,extbase',
     'clearcacheonload' => true,
     'state' => 'stable',
-    'version' => '18.6.1',
+    'version' => '18.8.0',
     'constraints' => [
         'depends' => [
             'typo3' => '6.0.0-9.2.99',

extension/qfq/api/delete.php

@@ -43,12 +43,12 @@ require_once(__DIR__ . '/../qfq/exceptions/CodeException.php');
  *
  * Description:
  *
- * Delete successfull.
+ * Delete successful.
  *  status = 'success'
  *  message = <message>
  *  redirect = 'client'
  *
- * Delete successfull.
+ * Delete successful.
  *  status = 'success'
  *  message = <message>
  *  redirect = 'url'

extension/qfq/api/dragAndDrop.php

@@ -59,12 +59,14 @@ try {
         $qfq = new QuickFormQuery(['bodytext' => '']);
 
         $data = $qfq->dragAndDrop();
+        $answer = array_merge($data, $answer);
 
         $answer[API_STATUS] = API_ANSWER_STATUS_SUCCESS;
         $answer[API_MESSAGE] = 'reorder: success';
 //    $answer[API_FORM_UPDATE] = $data[API_FORM_UPDATE];
 //    $answer[API_ELEMENT_UPDATE] = $data[API_ELEMENT_UPDATE];
 //    unset($answer[API_FORM_UPDATE][API_ELEMENT_UPDATE]);
+//        $answer[API_ELEMENT_UPDATE] = [ 'dynamic-125' => [ 'content' => 'latest news', 'attr' => [ 'title' => 'latest title'] ] ];
 
     } catch (qfq\UserFormException $e) {
         $answer[API_MESSAGE] = $e->formatMessage();

extension/qfq/api/file.php

@@ -13,6 +13,8 @@ use qfq;
 require_once(__DIR__ . '/../qfq/store/Store.php');
 require_once(__DIR__ . '/../qfq/Constants.php');
 require_once(__DIR__ . '/../qfq/File.php');
+require_once(__DIR__ . '/../qfq/exceptions/UserFormException.php');
+require_once(__DIR__ . '/../qfq/exceptions/CodeException.php');
 
 /**
  * Process File Upload - immediately when the the user selects a file.
@@ -23,7 +25,7 @@ require_once(__DIR__ . '/../qfq/File.php');
  *
  * Description:
  *
- * Upload successfull & File accepted by server.
+ * Upload successful & File accepted by server.
  *  status = 'success'
  *  message = <message>
  *

extension/qfq/qfq/Constants.php

@@ -258,6 +258,7 @@ const ERROR_DOWNLOAD_NOTHING_TO_DO = 1702;
 const ERROR_DOWNLOAD_UNEXPECTED_MIME_TYPE = 1703;
 const ERROR_DOWNLOAD_UNEXPECTED_NUMBER_OF_SOURCES = 1704;
 const ERROR_DOWNLOAD_FILE_NOT_READABLE = 1705;
+const ERROR_DOWNLOAD_FOPEN_BLOCKED = 1706;
 
 // Excel
 const ERROR_EXCEL_POSITION_ARGUMENT_EMPTY = 1800;
@@ -303,6 +304,7 @@ const ERROR_LANGUAGE_NOT_CONFIGURED_IN_QFQ = 2300;
 const ERROR_MISSING_SESSIONNAME = 2400;
 const ERROR_QFQ_SESSION_MISSING = 2401;
 const ERROR_SESSION_BROKEN_SCRIPT_PATH = 2402;
+const ERROR_MISSING_COOKIE = 2403;
 
 const ERROR_HTML2PDF_MISSING_CONFIG = 2500;
 const ERROR_HTML2PDF_WKHTML_NOT_EXECUTABLE = 2501;
@@ -1300,6 +1302,8 @@ const COLUMN_PAGEI = 'pagei';
 const COLUMN_PAGEN = 'pagen';
 const COLUMN_PAGES = 'pages';
 
+const COLUMN_YANK = 'yank';
+
 const COLUMN_PDF = 'pdf';
 const COLUMN_FILE = 'file';
 const COLUMN_ZIP = 'zip';
@@ -1311,9 +1315,10 @@ const COLUMN_STRIPTAGS = 'striptags';
 const COLUMN_MIME_TYPE = 'mimeType'; // Will also be used to identify equal named columns in upload record.
 const COLUMN_FILE_SIZE = 'fileSize'; // Will also be used to identify equal named columns in upload record.
 const COLUMN_IMPORT = "import";
-const COLUMN_EXCEL_STRING = 'XLSs';
-const COLUMN_EXCEL_NUMERIC = 'XLSn';
 const COLUMN_EXCEL_PLAIN = 'XLS';
+const COLUMN_EXCEL_NUMERIC = 'XLSn';
+const COLUMN_EXCEL_STRING = 'XLSs';
+const COLUMN_EXCEL_BASE64 = 'XLSb';
 
 const COLUMN_WRAP_TOKEN = '+';
 
@@ -1359,7 +1364,7 @@ const EXCEL_STRING2 = 'str';
 const EXCEL_STRING = 's';
 const EXCEL_FORMULA = 'f';
 const EXCEL_NUMERIC = 'n';
-const EXCEL_BOOL = 'b';
+const EXCEL_BASE64 = 'b';
 const EXCEL_NULL = 'null';
 const EXCEL_INLINE = 'inlineStr';
 const EXCEL_ERROR = 'e';
@@ -1551,4 +1556,5 @@ const DND_COLUMN_ORD_NEW = 'ordNew';
 const DND_DATA_DND_API = 'data-dnd-api';
 const DND_SUBRECORD_ID = 'dnd-subrecord-id'; // Internal qualifier used to communicate with dnd api for subrecord dnd
 const DND_SUBRECORD_FORM_ID = 'dnd-subrecord-form-id';
+const DND_ORD_HTML_ID_PREFIX = 'qfq-dnd-ord-id-';
 

extension/qfq/qfq/QuickFormQuery.php

@@ -319,6 +319,7 @@ class QuickFormQuery {
     private function doForm($formMode) {
         $data = '';
         $foundInStore = '';
+        $flagApiStructureReGroup=true;
 
         // Fill STORE_FORM
         if ($formMode === FORM_UPDATE || $formMode === FORM_SAVE) {
@@ -527,7 +528,8 @@ class QuickFormQuery {
                 $formAction->elements($recordId, $this->feSpecAction, FE_TYPE_BEFORE_LOAD);
 
                 $dragAndDrop = new DragAndDrop($this->formSpec);
-                $dragAndDrop->process();
+                $data = $dragAndDrop->process();
+                $flagApiStructureReGroup=false;
 
                 $formAction->elements($recordId, $this->feSpecAction, FE_TYPE_AFTER_LOAD);
                 break;
@@ -536,7 +538,7 @@ class QuickFormQuery {
                 throw new CodeException("This statement should never be reached", ERROR_CODE_SHOULD_NOT_HAPPEN);
         }
 
-        if (is_array($data)) {
+        if ($flagApiStructureReGroup && is_array($data)) {
             // $data['element-update']=...
             $data = $this->groupElementUpdateEntries($data);
         }
@@ -717,7 +719,7 @@ class QuickFormQuery {
                     unlink($file);
                 } else {
                     $form[FORM_LOG_FILE . '_' . $mode] = $file;
-                    $form[FORM_LOG_ACTIVE]=1;
+                    $form[FORM_LOG_ACTIVE] = 1;
                 }
             }
         }

extension/qfq/qfq/exceptions/AbstractException.php

@@ -35,9 +35,11 @@ class AbstractException extends \Exception {
     protected $line = '';
 
     /**
-     * $this->getMessage() might give a) a simple string or b) an JSON String.
+     * $this->getMessage() might give
+     *   a) a simple string, or
+     *   b) an JSON String.
      *
-     * JSON String: There are 3+1 different messages:
+     * If it is a JSON String: There are 3+1 different messages:
      *   [ERROR_MESSAGE_TO_USER] 'toUser' - shown in the client to the user - no details here!!!
      *   [ERROR_MESSAGE_SUPPORT] 'support' - help for the developer
      *   [ERROR_MESSAGE_OS] 'os' - message from the OS, like 'file not found'
@@ -54,6 +56,7 @@ class AbstractException extends \Exception {
         $arrShow = $this->messageArray;
         $htmlDebug = '';
         $arrDebugShow = array();
+        $editForm='';
 
         try {
             // In a very early stage, it might be possible that Store can't be initialized: take care not to use it.
@@ -117,15 +120,16 @@ class AbstractException extends \Exception {
                 // edit the broken form will be helpful.
                 $storeSystem = $store->getStore(STORE_SYSTEM);
                 if (!empty($storeSystem[SYSTEM_FORM])) {
-                    $arrDebugShow[EXCEPTION_EDIT_FORM] = $this->buildFormLink($storeSystem);
+                    $editForm = $this->buildFormLink($storeSystem);
                 }
 
-                $htmlDebug = OnArray::arrayToHtmlTable(OnArray::htmlentitiesOnArray(array_merge($arrMsg, $arrDebugShow)), 'Debug', EXCEPTION_TABLE_CLASS);
+                $htmlDebug = OnArray::arrayToHtmlTable(
+                    array_merge(OnArray::htmlentitiesOnArray(array_merge($arrMsg, $arrDebugShow)), [ 'Edit' => $editForm ]), 'Debug', EXCEPTION_TABLE_CLASS);
                 $htmlDebug = str_replace("\n", "<br>", $htmlDebug);
 
                 $arrDebugHiddenClean = OnArray::htmlentitiesOnArray($arrDebugHidden);
                 $arrDebugHiddenClean[EXCEPTION_STACKTRACE] = implode($arrTrace, '<br>');
-
+                $arrDebugHiddenClean[EXCEPTION_EDIT_FORM] = implode($arrTrace, '<br>');
                 $hidden = OnArray::arrayToHtmlTable($arrDebugHiddenClean, 'Details', EXCEPTION_TABLE_CLASS);
 
                 // Show / hide with just CSS: http://jsfiddle.net/t5Nf8/1/
@@ -204,6 +208,7 @@ class AbstractException extends \Exception {
             }
 
         } catch (\exception $e) {
+            // none, should rise up
         }
 
         return 'Form: ' . $linkForm . '&nbsp;&nbsp;  FormElement: ' . $linkFormElement;

extension/qfq/qfq/form/DragAndDrop.php

@@ -86,19 +86,34 @@ class DragAndDrop {
             return [];
         }
 
-        $this->reorder($this->formSpec[F_DRAG_AND_DROP_ORDER_SQL], $dragId, $setTo, $hoverId, $orderColumn, $orderInterval, $this->formSpec[F_TABLE_NAME]);
+        $data = $this->reorder($this->formSpec[F_DRAG_AND_DROP_ORDER_SQL], $dragId, $setTo, $hoverId, $orderColumn,
+            $orderInterval, $this->formSpec[F_TABLE_NAME]);
 
-        return [];
+        return $data;
     }
 
     /**
-     * @param array $rows
-     * @param $dragId
-     * @param $setTo
-     * @param $hoverId
-     * @param $orderColumn
-     * @param $orderInterval
-     * @param $tableName
+     * Calculate new ord values. The array rows$ contains the old order with
+     * [
+     *   [
+     *     [DND_COLUMN_ID] => 1,
+     *     [DND_COLUMN_ORD] => 10
+     *   ], [
+     *     [DND_COLUMN_ID] => 2,
+     *     [DND_COLUMN_ORD] => 20
+     *   ],
+     *   ...
+     * ]
+     *
+     * @param array $rows     Array with id/ord in the old order.
+     * @param int $dragId         Id of the element which has been drag'ed
+     * @param string $setTo          DND_SET_TO_BEFORE|DND_SET_TO_AFTER   Indicates if the drop zone is before or after the $hoverId
+     * @param int $hoverId        Id of element where the drag'ed element has been dropped on.
+     * @param string $orderColumn    Table column where to save the new calculated order.
+     * @param int $orderInterval  Order increment.
+     * @param string $tableName      Table name where to update the order records.
+     * @return array          Array with html-id references to update order values in the browser. Check PROTOCOL.md for 'element-update'.
+     *
      * @throws CodeException
      * @throws DbException
      * @throws UserFormException
@@ -106,6 +121,7 @@ class DragAndDrop {
     private function reorder(array $rows, $dragId, $setTo, $hoverId, $orderColumn, $orderInterval, $tableName) {
         $ord = $orderInterval;
         $ordDragOld = -1;
+        $data = array();
 
         // Reorder. Get index for 'drag' and 'hover'
         foreach ($rows as $key => $row) {
@@ -121,43 +137,52 @@ class DragAndDrop {
 
                 switch ($setTo) {
                     case DND_SET_TO_BEFORE:
-                        $this->setNewOrder($tableName, $orderColumn, $dragId, $ordDragOld, $ord);
+                        $data = $this->setNewOrder($tableName, $orderColumn, $dragId, $ordDragOld, $ord, $data);
                         $ord += $orderInterval;
-                        $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord);
+                        $data = $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord, $data);
                         break;
 
                     case DND_SET_TO_AFTER:
-                        $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord);
+                        $data = $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord, $data);
                         $ord += $orderInterval;
-                        $this->setNewOrder($tableName, $orderColumn, $dragId, $ordDragOld, $ord);
+                        $data = $this->setNewOrder($tableName, $orderColumn, $dragId, $ordDragOld, $ord, $data);
                         break;
 
                     default:
                         throw new CodeException('Unkown setTo string', $setTo, ERROR_UNKNOWN_TOKEN);
                 }
             } else {
-                $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord);
+                $data = $this->setNewOrder($tableName, $orderColumn, $row[DND_COLUMN_ID], $row[DND_COLUMN_ORD], $ord, $data);
             }
             $ord += $orderInterval;
         }
+
+        return $data;
     }
 
     /**
-     * @param $tableName
-     * @param $orderColumn
-     * @param $id
-     * @param $ordOld
-     * @param $ordNew
+     * @param string $tableName
+     * @param string $orderColumn
+     * @param int $id
+     * @param int $ordOld
+     * @param int $ordNew
+     * @param array $data
+     * @return array
      * @throws CodeException
      * @throws DbException
      * @throws UserFormException
      */
-    private function setNewOrder($tableName, $orderColumn, $id, $ordOld, $ordNew) {
+    private function setNewOrder($tableName, $orderColumn, $id, $ordOld, $ordNew, array $data) {
 
         if ($ordNew == $ordOld) {
-            return;
+            return $data;
         }
 
         $this->db->sql("UPDATE $tableName SET $orderColumn=? WHERE id=?", ROW_REGULAR, [$ordNew, $id]);
+
+        // Converting to string is necessary: JSON detects int else.
+        $data[API_ELEMENT_UPDATE][DND_ORD_HTML_ID_PREFIX . $id][API_ELEMENT_CONTENT] = (string)$ordNew;
+
+        return $data;
     }
 }
\ No newline at end of file

extension/qfq/qfq/helper/DownloadPage.php

@@ -8,6 +8,8 @@
 
 namespace qfq;
 
+require_once(__DIR__ . '/../exceptions/UserReportException.php');
+
 
 /**
  * Class DownloadPage
@@ -22,6 +24,7 @@ class DownloadPage {
      * @param string $page - Full URL like 'https://example.com' or T3 specific like 'export&s=badcaffee1234'
      * @param string $baseUrl - baserUrl from Configfile
      * @return bool|string
+     * @throws UserReportException
      */
     public static function getContent($page, $baseUrl) {
 
@@ -37,14 +40,34 @@ class DownloadPage {
             $page = $baseUrl . $page;
         }
 
+        // Check php.ini
+        if (ini_get('allow_url_fopen') != '1') {
+            throw new UserReportException("php.ini: 'allow_url_fopen' is off and needs to be open to get '$page''", ERROR_DOWNLOAD_FOPEN_BLOCKED);
+        }
+
+        if (!isset($_COOKIE[SESSION_NAME])) {
+            throw new UserReportException('Error: missing QFQ cookie - allow cookies in your browser', ERROR_MISSING_COOKIE);
+        }
+
         // Download page
         $ctx = array(