Merge branch '4922-excel-import' into 'master'

4922 excel import

See merge request !73

extension/Documentation/Manual.rst

@@ -3470,6 +3470,21 @@ See also `downloadButton`_ to offer a download of an uploaded file.
 
   * fileSplit, fileDestinationSplit, tableNameSplit: see split-pdf-upload_
 
+  * Excel Import: QFQ offers functionality to directly import excel data into the database. This functionality can
+    optionally be combined with saving the file by using the above parameters like `fileDestination`.
+
+    * *importToTable*: <mariadb.tablename> - **Required**. Providing this parameter activates the import. If the table
+      doesn't exist, it will be created.
+    * *importToColumn*: <col1>,<col2>,... - If none provided, the Excel column names A, B, ... are used. Note: These
+      have to match the table's column names if the table already exists.
+    * *importRegion*: [tab],[startColumn],[startRow],[endColumn],[endRow]|... - All parts are optional (default:
+      entire 1st sheet). Tab can either be given as an index (1-based) or a name. start/endColumn can be given either
+      numerically (1, 2, ...) or by column name (A, B, ...). Note that you can specify several regions to import.
+    * *importMode*: `append` (default) | `replace` - The data is either appended or replace in the specified table.
+    * *importType*: `auto` (default) | `xls` | `xlsx` | `ods` | `csv` - Define what kind of data should be expected by the
+      Spreadsheet Reader. `auto` should work fine in most cases.
+
+
 Immediately after the upload finished (before the user press save), the file will be checked on the server for it's
 content or file extension (see 'accept').
 

extension/qfq/qfq/Constants.php

@@ -246,6 +246,7 @@ const ERROR_UPLOAD_GET_MIME_TYPE = 1503;
 const ERROR_UPLOAD_UNKNOWN_ACTION = 1504;
 const ERROR_NO_TARGET_PATH_FILE_NAME = 1505;
 const ERROR_UPLOAD_FILES_BROKEN = 1506;
+const ERROR_UNKNOWN_EXCEL_IMPORT_TYPE = 1507;
 
 // LDAP / typeahead
 const ERROR_LDAP_CONNECT = 1600;
@@ -976,6 +977,20 @@ const FE_FILE_SPLIT_SVG = 'svg';
 const FE_FILE_SPLIT_TABLE_NAME = 'tableNameSplit';
 const FE_FILE_DOWNLOAD_BUTTON = 'downloadButton';
 
+// Excel Import
+const FE_IMPORT_TO_TABLE = 'importToTable';
+const FE_IMPORT_TO_COLUMNS = 'importToColumns';
+const FE_IMPORT_REGION = 'importRegion';
+const FE_IMPORT_MODE = 'importMode';
+const FE_IMPORT_MODE_APPEND = 'append';
+const FE_IMPORT_MODE_REPLACE = 'replace';
+const FE_IMPORT_TYPE = 'importType';
+const FE_IMPORT_TYPE_AUTO = 'auto';
+const FE_IMPORT_TYPE_XLS = 'xls';
+const FE_IMPORT_TYPE_XLSX = 'xlsx';
+const FE_IMPORT_TYPE_ODS = 'ods';
+const FE_IMPORT_TYPE_CSV = 'csv';
+
 const FE_IMAGE_SOURCE = 'imageSource'; // Image source for a fabric element
 const FE_SQL_VALIDATE = 'sqlValidate'; // Action: Query to validate form load
 const FE_EXPECT_RECORDS = 'expectRecords'; // Action: expected number of rows of FE_SQL_VALIDATE

extension/qfq/qfq/Save.php

@@ -13,6 +13,7 @@ require_once(__DIR__ . '/store/Sip.php');
 require_once(__DIR__ . '/Constants.php');
 require_once(__DIR__ . '/Evaluate.php');
 require_once(__DIR__ . '/helper/HelperFile.php');
+require_once(__DIR__ . '/helper/OnArray.php');
 //require_once(__DIR__ . '/../qfq/exceptions/UserException.php');
 //require_once(__DIR__ . '/../qfq/exceptions/CodeException.php');
 //require_once(__DIR__ . '/../qfq/exceptions/DbException.php');
@@ -544,6 +545,8 @@ class Save {
      * @throws DbException
      * @throws UserFormException
      * @throws UserReportException
+     * @throws \PhpOffice\PhpSpreadsheet\Exception
+     * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
      * @internal param $recordId
      */
     private function doUpload($formElement, $sipUpload, Sip $sip, &$modeUpload) {
@@ -556,6 +559,13 @@ class Save {
             return false;
         }
 
+        if (isset($formElement[FE_IMPORT_TO_TABLE]) && isset($statusUpload[FILES_TMP_NAME])) {
+            // Import
+            $tmpFile = Support::extendFilename($statusUpload[FILES_TMP_NAME], UPLOAD_CACHED);
+            $this->doImport($formElement, $tmpFile);
+        }
+
+        // Upload
         // Take care the necessary target directories exist.
         $cwd = getcwd();
         $sitePath = $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM);
@@ -582,7 +592,10 @@ class Save {
             $modeUpload = $flagDelete ? UPLOAD_MODE_DELETEOLD : UPLOAD_MODE_UNCHANGED;
         }
 
-        $pathFileName = $this->copyUploadFile($formElement, $statusUpload);
+        // skip uploading the file if this is an import without a specified file destination
+        if (!isset($formElement[FE_IMPORT_TO_TABLE]) || isset($formElement[FE_FILE_DESTINATION])) {
+            $pathFileName = $this->copyUploadFile($formElement, $statusUpload);
+        }
 
         chdir($cwd);
 
@@ -592,6 +605,128 @@ class Save {
         return $pathFileName;
     }
 
+    /**
+     * @param $formElement
+     * @throws CodeException
+     * @throws DbException
+     * @throws UserFormException
+     * @throws \PhpOffice\PhpSpreadsheet\Exception
+     * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
+     */
+    private function doImport($formElement, $fileName) {
+        Support::setIfNotSet($formElement, FE_IMPORT_TYPE, FE_IMPORT_TYPE_AUTO);
+        switch ($formElement[FE_IMPORT_TYPE]) {
+            case FE_IMPORT_TYPE_AUTO:
+                $spreadsheet = \PhpOffice\PhpSpreadsheet\IOFactory::load($fileName);
+                break;
+
+            case FE_IMPORT_TYPE_XLS:
+            case FE_IMPORT_TYPE_XLSX:
+            case FE_IMPORT_TYPE_CSV:
+            case FE_IMPORT_TYPE_ODS:
+                $inputFileType = ucfirst($formElement[FE_IMPORT_TYPE]);
+                $reader = \PhpOffice\PhpSpreadsheet\IOFactory::createReader($inputFileType);
+                $spreadsheet = $reader->load($fileName);
+                break;
+
+            default:
+                throw new UserFormException("Unknown Excel import type: '" . $formElement[FE_IMPORT_TYPE] . "'.",
+                    ERROR_UNKNOWN_EXCEL_IMPORT_TYPE);
+        }
+
+        $tableName = $formElement[FE_IMPORT_TO_TABLE];
+        $regions = OnArray::trimArray(explode('|', $formElement[FE_IMPORT_REGION] ?? ''));
+        $columnNames = OnArray::trimArray(explode(',', $formElement[FE_IMPORT_TO_COLUMNS]));
+        $importMode = $formElement[FE_IMPORT_MODE] ?? FE_IMPORT_MODE_APPEND;
+
+        foreach ($regions as $region) {
+            // region: tab, startColumn, startRow, endColumn, endRow
+            $region = OnArray::trimArray(explode(',', $region));
+            $tab = 1;
+            if (!empty($region[0])) {
+                $tab = $region[0];
+            }
+
+            try {
+                if (is_numeric($tab)) {
+                    $worksheet = $spreadsheet->getSheet($tab - 1); // 0-based
+                } else {
+                    $worksheet = $spreadsheet->getSheetByName($tab);
+                    if ($worksheet === null) {
+                        throw new \PhpOffice\PhpSpreadsheet\Exception(
+                            "No sheet with the name '$tab' could be found."
+                        );
+                    }
+                }
+            } catch (\PhpOffice\PhpSpreadsheet\Exception $e) {
+                throw new UserFormException($e->getMessage());
+            }
+
+            // Set up requested region
+            $columnStart = '1';
+            $columnEnd = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($worksheet->getHighestColumn());
+            $rowStart = 1;
+            $rowEnd = $worksheet->getHighestRow();
+            if (!empty($region[1])) { // startColumn
+                if (!is_numeric($region[1])) $region[1] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[1]);
+                if ($region[1] >= $columnStart && $region[1] <= $columnEnd) {
+                    $columnStart = $region[1];
+                }
+            }
+            if (!empty($region[3])) { // endColumn
+                if (!is_numeric($region[3])) $region[3] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[3]);
+                if ($region[3] >= $columnStart && $region[3] <= $columnEnd) {
+                    $columnEnd = $region[3];
+                }
+            }
+            if (!empty($region[2]) && $region[2] >= $rowStart && $region[2] <= $rowEnd) {
+                $rowStart = $region[2];
+            }
+            if (!empty($region[4]) && $region[4] >= $rowStart && $region[4] <= $rowEnd) {
+                $rowEnd = $region[4];
+            }
+            // Read the specified region
+            $rangeStr = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnStart) . $rowStart . ':' .
+                \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnEnd) . $rowEnd;
+            $worksheetData = $worksheet->rangeToArray($rangeStr, '', true, false);
+
+            $columnDefinitionArr = [];
+            $columnListArr = [];
+            for ($column = $columnStart; $column <= $columnEnd; ++$column) {
+                if (!empty($columnNames[$column - $columnStart])) {
+                    $columnName = $columnNames[$column - $columnStart];
+                } else {
+                    $columnName = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($column);
+                }
+                $columnDefinitionArr[] = "`$columnName`   TEXT       NOT NULL  DEFAULT ''";
+                $columnListArr[] = "$columnName";
+            }
+
+            // SQL time!
+            $createTableSql = "CREATE TABLE IF NOT EXISTS `$tableName` (" .
+                "`id`        INT(11)    NOT NULL  AUTO_INCREMENT," .
+                implode(', ', $columnDefinitionArr) . ',' .
+                "`modified`  TIMESTAMP  NOT NULL  DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," .
+                "`created`   DATETIME   NOT NULL  DEFAULT CURRENT_TIMESTAMP," .
+                "PRIMARY KEY (`id`) )" .
+                "ENGINE = InnoDB  DEFAULT CHARSET = utf8 AUTO_INCREMENT = 0;";
+            $this->db->sql($createTableSql);
+
+            if ($importMode === FE_IMPORT_MODE_REPLACE) {
+                $this->db->sql("TRUNCATE $tableName");
+                $importMode = FE_IMPORT_MODE_APPEND;
+            }
+
+            // Import the data
+            foreach ($worksheetData AS $rowIndex => $row) {
+                $columnList = implode(',', $columnListArr);
+                $paramPlaceholders = str_repeat('?,', count($worksheetData[0]) - 1) . '?';
+                $insertSql = "INSERT INTO `$tableName` ($columnList) VALUES ($paramPlaceholders)";
+                $this->db->sql($insertSql, ROW_REGULAR, $row);
+            }
+        }
+    }
+
     /**
      * Copy uploaded file from temporary location to final location.
      *

extension/qfq/qfq/helper/HelperFormElement.php

@@ -410,7 +410,7 @@ EOF;
     }
 
     /**
-     * Returns $maxLenght if greater than 0, else FE_TEMPLATE_GROUP_DEFAULT_MAX_LENGTH
+     * Returns $maxLength if greater than 0, else FE_TEMPLATE_GROUP_DEFAULT_MAX_LENGTH
      *
      * @param $maxLength
      *