Skip to content

GitLab

Commit 6075fad6 authored by Elias Villiger's avatar Elias Villiger
Browse files

Feature #4922 - Finalize and document Excel Import

parent 022ca809
Pipeline #865 passed with stage
in 1 minute and 47 seconds
Hide whitespace changes
Inline Side-by-side
extension/Documentation/Manual.rst
View file @ 6075fad6
......@@ -3470,6 +3470,21 @@ See also `downloadButton`_ to offer a download of an uploaded file.
* fileSplit, fileDestinationSplit, tableNameSplit: see split-pdf-upload_
* Excel Import: QFQ offers functionality to directly import excel data into the database. This functionality can
optionally be combined with saving the file by using the above parameters like `fileDestination`.
* *importToTable*: <mariadb.tablename> - **Required**. Providing this parameter activates the import. If the table
doesn't exist, it will be created.
* *importToColumn*: <col1>,<col2>,... - If none provided, the Excel column names A, B, ... are used. Note: These
have to match the table's column names if the table already exists.
* *importRegion*: [tab],[startColumn],[startRow],[endColumn],[endRow]|... - All parts are optional (default:
entire 1st sheet). Tab can either be given as an index (1-based) or a name. start/endColumn can be given either
numerically (1, 2, ...) or by column name (A, B, ...). Note that you can specify several regions to import.
* *importMode*: `append` (default) | `replace` - The data is either appended or replace in the specified table.
* *importType*: `auto` (default) | `xls` | `xlsx` | `ods` | `csv` - Define what kind of data should be expected by the
Spreadsheet Reader. `auto` should work fine in most cases.
Immediately after the upload finished (before the user press save), the file will be checked on the server for it's
content or file extension (see 'accept').
......
extension/qfq/qfq/Constants.php
View file @ 6075fad6
......@@ -246,6 +246,7 @@ const ERROR_UPLOAD_GET_MIME_TYPE = 1503;
const ERROR_UPLOAD_UNKNOWN_ACTION = 1504;
const ERROR_NO_TARGET_PATH_FILE_NAME = 1505;
const ERROR_UPLOAD_FILES_BROKEN = 1506;
const ERROR_UNKNOWN_EXCEL_IMPORT_TYPE = 1507;
// LDAP / typeahead
const ERROR_LDAP_CONNECT = 1600;
......@@ -978,6 +979,12 @@ const FE_IMPORT_REGION = 'importRegion';
const FE_IMPORT_MODE = 'importMode';
const FE_IMPORT_MODE_APPEND = 'append';
const FE_IMPORT_MODE_REPLACE = 'replace';
const FE_IMPORT_TYPE = 'importType';
const FE_IMPORT_TYPE_AUTO = 'auto';
const FE_IMPORT_TYPE_XLS = 'xls';
const FE_IMPORT_TYPE_XLSX = 'xlsx';
const FE_IMPORT_TYPE_ODS = 'ods';
const FE_IMPORT_TYPE_CSV = 'csv';
const FE_IMAGE_SOURCE = 'imageSource'; // Image source for a fabric element
const FE_SQL_VALIDATE = 'sqlValidate'; // Action: Query to validate form load
......
extension/qfq/qfq/Save.php
View file @ 6075fad6
......@@ -558,137 +558,172 @@ class Save {
return false;
}
if (isset($formElement[FE_IMPORT_TO_TABLE])) { // Import
// Read tmp file with Spreadsheet reader
if (isset($formElement[FE_IMPORT_TO_TABLE])) {
// Import
$tmpFile = Support::extendFilename($statusUpload[FILES_TMP_NAME], UPLOAD_CACHED);
$spreadsheet = \PhpOffice\PhpSpreadsheet\IOFactory::load($tmpFile);
$tableName = $formElement[FE_IMPORT_TO_TABLE];
$regions = explode('|', $formElement[FE_IMPORT_REGION] ?? '');
$columnNames = explode(',', $formElement[FE_IMPORT_TO_COLUMNS]);
$importMode = $formElement[FE_IMPORT_MODE] ?? FE_IMPORT_MODE_APPEND;
foreach ($regions as $region) {
// region: tab, startColumn, startRow, endColumn, endRow
$region = explode(',', $region);
$tab = 1;
if (!empty($region[0])) {
$tab = $region[0];
}
$this->doImport($formElement, $tmpFile);
}
try {
if (is_numeric($tab)) {
$worksheet = $spreadsheet->getSheet($tab - 1); // 0-based
} else {
$worksheet = $spreadsheet->getSheetByName($tab);
if ($worksheet === null) {
throw new \PhpOffice\PhpSpreadsheet\Exception(
"No sheet with the name '$tab' could be found."
);
}
}
} catch (\PhpOffice\PhpSpreadsheet\Exception $e) {
throw new UserFormException($e->getMessage());
}
// Upload
// Take care the necessary target directories exist.
$cwd = getcwd();
$sitePath = $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM);
if ($cwd === false || $sitePath === false || !chdir($sitePath)) {
throw new UserFormException("getcwd() failed or SITE_PATH undefined or chdir('$sitePath') failed.", ERROR_IO_CHDIR);
}
// Set up requested region
$columnStart = '1';
$columnEnd = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($worksheet->getHighestColumn());
$rowStart = 1;
$rowEnd = $worksheet->getHighestRow();
if (!empty($region[1])) { // startColumn
if (!is_numeric($region[1])) $region[1] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[1]);
if ($region[1] >= $columnStart && $region[1] <= $columnEnd) {
$columnStart = $region[1];
}
}
if (!empty($region[3])) { // endColumn
if (!is_numeric($region[3])) $region[3] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[3]);
if ($region[3] >= $columnStart && $region[3] <= $columnEnd) {
$columnEnd = $region[3];
}
}
if (!empty($region[2]) && $region[2] >= $rowStart && $region[2] <= $rowEnd) {
$rowStart = $region[2];
// Delete existing old file.
if (isset($statusUpload[FILES_FLAG_DELETE]) && $statusUpload[FILES_FLAG_DELETE] == '1') {
$arr = $sip->getVarsFromSip($sipUpload);
$oldFile = $arr[EXISTING_PATH_FILE_NAME];
if (file_exists($oldFile)) {
if (!unlink($oldFile)) {
throw new UserFormException('Unlink file failed: ' . $oldFile, ERROR_IO_UNLINK);
}
if (!empty($region[4]) && $region[4] >= $rowStart && $region[4] <= $rowEnd) {
$rowEnd = $region[4];
}
// Read the specified region
$rangeStr = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnStart) . $rowStart . ':' .
\PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnEnd) . $rowEnd;
$worksheetData = $worksheet->rangeToArray($rangeStr, '', true, false);
$columnDefinitionArr = [];
$columnListArr = [];
for ($column = $columnStart; $column <= $columnEnd; ++$column) {
if (!empty($columnNames[$column - $columnStart])) {
$columnName = $columnNames[$column - $columnStart];
} else {
$columnName = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($column);
}
$flagDelete = ($oldFile != '');
}
// Set $modeUpload
if (isset($statusUpload[FILES_TMP_NAME]) && $statusUpload[FILES_TMP_NAME] != '') {
$modeUpload = $flagDelete ? UPLOAD_MODE_DELETEOLD_NEW : UPLOAD_MODE_NEW;
} else {
$modeUpload = $flagDelete ? UPLOAD_MODE_DELETEOLD : UPLOAD_MODE_UNCHANGED;
}
// skip uploading the file if this is an import without a specified file destination
if (!isset($formElement[FE_IMPORT_TO_TABLE]) || isset($formElement[FE_FILE_DESTINATION])) {
$pathFileName = $this->copyUploadFile($formElement, $statusUpload);
}
chdir($cwd);
// Delete current used uniq SIP
$this->store->setVar($sipUpload, array(), STORE_EXTRA);
return $pathFileName;
}
/**
* @param $formElement
* @throws CodeException
* @throws DbException
* @throws UserFormException
* @throws \PhpOffice\PhpSpreadsheet\Exception
* @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
*/
private function doImport($formElement, $fileName) {
Support::setIfNotSet($formElement, FE_IMPORT_TYPE, FE_IMPORT_TYPE_AUTO);
switch ($formElement[FE_IMPORT_TYPE]) {
case FE_IMPORT_TYPE_AUTO:
$spreadsheet = \PhpOffice\PhpSpreadsheet\IOFactory::load($fileName);
break;
case FE_IMPORT_TYPE_XLS:
case FE_IMPORT_TYPE_XLSX:
case FE_IMPORT_TYPE_CSV:
case FE_IMPORT_TYPE_ODS:
$inputFileType = ucfirst($formElement[FE_IMPORT_TYPE]);
$reader = \PhpOffice\PhpSpreadsheet\IOFactory::createReader($inputFileType);
$spreadsheet = $reader->load($fileName);
break;
default:
throw new UserFormException("Unknown Excel import type: '" . $formElement[FE_IMPORT_TYPE] . "'.",
ERROR_UNKNOWN_EXCEL_IMPORT_TYPE);
}
$tableName = $formElement[FE_IMPORT_TO_TABLE];
$regions = explode('|', $formElement[FE_IMPORT_REGION] ?? '');
$columnNames = explode(',', $formElement[FE_IMPORT_TO_COLUMNS]);
$importMode = $formElement[FE_IMPORT_MODE] ?? FE_IMPORT_MODE_APPEND;
foreach ($regions as $region) {
// region: tab, startColumn, startRow, endColumn, endRow
$region = explode(',', $region);
$tab = 1;
if (!empty($region[0])) {
$tab = $region[0];
}
try {
if (is_numeric($tab)) {
$worksheet = $spreadsheet->getSheet($tab - 1); // 0-based
} else {
$worksheet = $spreadsheet->getSheetByName($tab);
if ($worksheet === null) {
throw new \PhpOffice\PhpSpreadsheet\Exception(
"No sheet with the name '$tab' could be found."
);
}
$columnDefinitionArr[] = "`$columnName` TEXT NOT NULL DEFAULT ''";
$columnListArr[] = "$columnName";
}
// SQL time!
$createTableSql = "CREATE TABLE IF NOT EXISTS `$tableName` (" .
"`id` INT(11) NOT NULL AUTO_INCREMENT," .
implode(', ', $columnDefinitionArr) . ',' .
"`modified` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," .
"`created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP," .
"PRIMARY KEY (`id`) )" .
"ENGINE = InnoDB DEFAULT CHARSET = utf8 AUTO_INCREMENT = 0;";
$this->db->sql($createTableSql);
if ($importMode === FE_IMPORT_MODE_REPLACE) {
$this->db->sql("TRUNCATE $tableName");
$importMode = FE_IMPORT_MODE_APPEND;
}
} catch (\PhpOffice\PhpSpreadsheet\Exception $e) {
throw new UserFormException($e->getMessage());
}
// Import the data
foreach ($worksheetData AS $rowIndex => $row) {
$columnList = implode(',', $columnListArr);
$paramPlaceholders = str_repeat('?,', count($worksheetData[0]) - 1) . '?';
$insertSql = "INSERT INTO `$tableName` ($columnList) VALUES ($paramPlaceholders)";
$this->db->sql($insertSql, ROW_REGULAR, $row);
// Set up requested region
$columnStart = '1';
$columnEnd = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($worksheet->getHighestColumn());
$rowStart = 1;
$rowEnd = $worksheet->getHighestRow();
if (!empty($region[1])) { // startColumn
if (!is_numeric($region[1])) $region[1] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[1]);
if ($region[1] >= $columnStart && $region[1] <= $columnEnd) {
$columnStart = $region[1];
}
}
} else { // Upload
// Take care the necessary target directories exist.
$cwd = getcwd();
$sitePath = $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM);
if ($cwd === false || $sitePath === false || !chdir($sitePath)) {
throw new UserFormException("getcwd() failed or SITE_PATH undefined or chdir('$sitePath') failed.", ERROR_IO_CHDIR);
}
// Delete existing old file.
if (isset($statusUpload[FILES_FLAG_DELETE]) && $statusUpload[FILES_FLAG_DELETE] == '1') {
$arr = $sip->getVarsFromSip($sipUpload);
$oldFile = $arr[EXISTING_PATH_FILE_NAME];
if (file_exists($oldFile)) {
if (!unlink($oldFile)) {
throw new UserFormException('Unlink file failed: ' . $oldFile, ERROR_IO_UNLINK);
}
if (!empty($region[3])) { // endColumn
if (!is_numeric($region[3])) $region[3] = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($region[3]);
if ($region[3] >= $columnStart && $region[3] <= $columnEnd) {
$columnEnd = $region[3];
}
$flagDelete = ($oldFile != '');
}
// Set $modeUpload
if (isset($statusUpload[FILES_TMP_NAME]) && $statusUpload[FILES_TMP_NAME] != '') {
$modeUpload = $flagDelete ? UPLOAD_MODE_DELETEOLD_NEW : UPLOAD_MODE_NEW;
} else {
$modeUpload = $flagDelete ? UPLOAD_MODE_DELETEOLD : UPLOAD_MODE_UNCHANGED;
if (!empty($region[2]) && $region[2] >= $rowStart && $region[2] <= $rowEnd) {
$rowStart = $region[2];
}
if (!empty($region[4]) && $region[4] >= $rowStart && $region[4] <= $rowEnd) {
$rowEnd = $region[4];
}
// Read the specified region
$rangeStr = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnStart) . $rowStart . ':' .
\PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($columnEnd) . $rowEnd;
$worksheetData = $worksheet->rangeToArray($rangeStr, '', true, false);
$columnDefinitionArr = [];
$columnListArr = [];
for ($column = $columnStart; $column <= $columnEnd; ++$column) {
if (!empty($columnNames[$column - $columnStart])) {
$columnName = $columnNames[$column - $columnStart];
} else {
$columnName = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::stringFromColumnIndex($column);
}
$columnDefinitionArr[] = "`$columnName` TEXT NOT NULL DEFAULT ''";
$columnListArr[] = "$columnName";
}
$pathFileName = $this->copyUploadFile($formElement, $statusUpload);
// SQL time!
$createTableSql = "CREATE TABLE IF NOT EXISTS `$tableName` (" .
"`id` INT(11) NOT NULL AUTO_INCREMENT," .
implode(', ', $columnDefinitionArr) . ',' .
"`modified` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," .
"`created` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP," .
"PRIMARY KEY (`id`) )" .
"ENGINE = InnoDB DEFAULT CHARSET = utf8 AUTO_INCREMENT = 0;";
$this->db->sql($createTableSql);
if ($importMode === FE_IMPORT_MODE_REPLACE) {
$this->db->sql("TRUNCATE $tableName");
$importMode = FE_IMPORT_MODE_APPEND;
}
chdir($cwd);
// Import the data
foreach ($worksheetData AS $rowIndex => $row) {
$columnList = implode(',', $columnListArr);
$paramPlaceholders = str_repeat('?,', count($worksheetData[0]) - 1) . '?';
$insertSql = "INSERT INTO `$tableName` ($columnList) VALUES ($paramPlaceholders)";
$this->db->sql($insertSql, ROW_REGULAR, $row);
}
}
// Delete current used uniq SIP
$this->store->setVar($sipUpload, array(), STORE_EXTRA);
return $pathFileName;
}
/**
......
extension/qfq/qfq/helper/HelperFormElement.php
View file @ 6075fad6
......@@ -409,7 +409,7 @@ EOF;
}
/**
* Returns $maxLenght if greater than 0, else FE_TEMPLATE_GROUP_DEFAULT_MAX_LENGTH
* Returns $maxLength if greater than 0, else FE_TEMPLATE_GROUP_DEFAULT_MAX_LENGTH
*
* @param $maxLength
*
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment