From fca9143617d00e1eb0ca8eab76bea9e7d8b43e49 Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Sun, 14 Jun 2020 16:01:56 +0200 Subject: [PATCH 1/9] Add CWD to exception message. --- extension/Classes/Core/Constants.php | 1 + extension/Classes/Core/Exception/AbstractException.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/extension/Classes/Core/Constants.php b/extension/Classes/Core/Constants.php index 95809ecb..3e03467c 100644 --- a/extension/Classes/Core/Constants.php +++ b/extension/Classes/Core/Constants.php @@ -1911,6 +1911,7 @@ const EXCEPTION_MESSAGE_DEBUG = SYSTEM_MESSAGE_DEBUG; // Will only be shown as const EXCEPTION_FILE = 'File'; const EXCEPTION_LINE = 'Line'; +const EXCEPTION_CWD = 'CWD'; const EXCEPTION_STACKTRACE = 'Stacktrace'; const EXCEPTION_IP_ADDRESS = 'IP Address'; const EXCEPTION_QFQ_COOKIE = 'QFQ Cookie'; diff --git a/extension/Classes/Core/Exception/AbstractException.php b/extension/Classes/Core/Exception/AbstractException.php index bc2b28a2..55e101e8 100644 --- a/extension/Classes/Core/Exception/AbstractException.php +++ b/extension/Classes/Core/Exception/AbstractException.php @@ -103,11 +103,11 @@ class AbstractException extends \Exception { if (isset($arrMsg[ERROR_MESSAGE_HTTP_STATUS])) { $this->httpStatusCode = $arrMsg[ERROR_MESSAGE_HTTP_STATUS]; } - } $arrDebugHidden[EXCEPTION_FILE] = $this->getFile(); $arrDebugHidden[EXCEPTION_LINE] = $this->getLine(); + $arrDebugHidden[EXCEPTION_CWD] = getcwd(); $arrTrace = $this->getExtensionTraceAsArray(); if ($store !== null) { -- GitLab From 8781e0c226f7fe1c1e3947fbbfcfdeceb1892eaf Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Sun, 14 Jun 2020 16:27:12 +0200 Subject: [PATCH 2/9] Refs #10751. Add '2>&1' to merge stderr in stdout to see problems in qfq.log --- extension/Classes/Core/Report/Download.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/Classes/Core/Report/Download.php b/extension/Classes/Core/Report/Download.php index 97c146f7..62c917b8 100644 --- a/extension/Classes/Core/Report/Download.php +++ b/extension/Classes/Core/Report/Download.php @@ -200,7 +200,7 @@ class Download { } // img2pdf --pagesize A4 -o out.pdf *.jpg - $cmd = $this->img2Pdf . ' --pagesize A4 -o ' . $filePdf . ' ' . escapeshellarg($fileImage); + $cmd = $this->img2Pdf . ' --pagesize A4 -o ' . $filePdf . ' ' . escapeshellarg($fileImage) . ' 2>&1'; if ($this->downloadDebugLog != '') { Logger::logMessage("Download: $cmd", $this->downloadDebugLog); @@ -266,7 +266,7 @@ class Download { if ($rcQpdf != 0) { // Try 2: via 'gs -sDEVICE=pdfwrite' - $cmdGs = $this->gs . " -sDEVICE=pdfwrite -dNOPAUSE -sOutputFile=\"$file\" -- \"$backup\""; + $cmdGs = $this->gs . " -sDEVICE=pdfwrite -dNOPAUSE -sOutputFile=\"$file\" -- \"$backup\" 2>&1"; exec($cmdGs, $outputGs, $rcGs); if ($rcGs != 0) { -- GitLab From 9893fe32f0e8cdccaca3fa81bd35282ed010ad6e Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Tue, 16 Jun 2020 12:55:50 +0200 Subject: [PATCH 3/9] Manual.rst: Fix typo, add note to install img2pdf --- Documentation/Installation.rst | 5 +++-- Documentation/Release.rst | 1 + extension/Classes/Core/Form/FormAction.php | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Documentation/Installation.rst b/Documentation/Installation.rst index 705eb760..463e928f 100644 --- a/Documentation/Installation.rst +++ b/Documentation/Installation.rst @@ -42,7 +42,7 @@ The following features are only tested / supported on linux hosts: * General: QFQ is coded to run on Linux hosts, preferable on Debian derivates like Ubuntu. * HTML to PDF conversion - command `wkhtmltopdf`. * Concatenation of PDF files - command `pdfunite`. -* Convert of imges to PDF files - command `img2pdf`. +* Convert of images to PDF files - command `img2pdf`. * PDF decrypt (used for merge with pdfunite) - command `qpdf`. * PDF decrypt (used for merge with pdfunite) - command `gs` - in case `qpdf` is not successful. * Mime type detection for uploads - command `file`. @@ -65,7 +65,8 @@ For the :ref:`download` function, the programs `img2pdf`, `pdfunite`, `qpdf`, `g Preparation for Ubuntu:: sudo apt install php-intl - sudo apt install poppler-utils libxrender1 file pdf2svg qpdf ghostscript # for file upload, PDF and 'HTML to PDF' (wkhtmltopdf), PDF split + # for file upload, PDF and 'HTML to PDF' (wkhtmltopdf), PDF split + sudo apt install poppler-utils libxrender1 file pdf2svg qpdf ghostscript img2pdf sudo apt install inkscape imagemagick # to render thumbnails .. _wkhtml: diff --git a/Documentation/Release.rst b/Documentation/Release.rst index 836e62af..61e3977d 100644 --- a/Documentation/Release.rst +++ b/Documentation/Release.rst @@ -48,6 +48,7 @@ Notes in filenames and wkhtml commandline options (like header/footer). * Migrate documentation from T3 to ReadTheDocs.io - looks older but 'search' is much more better. New: chapters separated in individual files. + * For the image to PDF feature, installation of `img2pdf` is required (please check `preparation`_). Features ^^^^^^^^ diff --git a/extension/Classes/Core/Form/FormAction.php b/extension/Classes/Core/Form/FormAction.php index a66e5455..c452db09 100644 --- a/extension/Classes/Core/Form/FormAction.php +++ b/extension/Classes/Core/Form/FormAction.php @@ -328,16 +328,17 @@ class FormAction { // If there is at least one record count given, who matches: return 'check succeeded' $countRecordsArr = explode(',', $expect); - foreach ($countRecordsArr AS $count) { + foreach ($countRecordsArr as $count) { if (count($result) == $count) { - return; // check succesfully passed + return; // check successfully passed } } $msg = $this->evaluate->parse($fe[FE_MESSAGE_FAIL]); // Replace possible dynamic parts // Throw user error message - throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => $msg, ERROR_MESSAGE_TO_DEVELOPER => 'validate() failed']), ERROR_REPORT_FAILED_ACTION); + throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => $msg + , ERROR_MESSAGE_TO_DEVELOPER => 'validate() failed']), ERROR_REPORT_FAILED_ACTION); } -- GitLab From 4f2889efddbc9aabdd405e38b566f989117b8fa1 Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Sun, 21 Jun 2020 14:09:53 +0200 Subject: [PATCH 4/9] Manual.rst: Removed never customized template file Links.rst. --- Documentation/Form.rst | 8 +++--- Documentation/Links.rst | 58 ----------------------------------------- Documentation/index.rst | 1 - 3 files changed, 4 insertions(+), 63 deletions(-) delete mode 100644 Documentation/Links.rst diff --git a/Documentation/Form.rst b/Documentation/Form.rst index b9a47f44..6157a9d3 100644 --- a/Documentation/Form.rst +++ b/Documentation/Form.rst @@ -586,7 +586,7 @@ The `mode` is given via (in this priority): Mode ;;;; -* *standard*: +* **standard**: * The form will behave like defined in the form editor. * Missing required values will a) be indicated and b) block saving the record. @@ -2027,7 +2027,7 @@ FormElement.parameter * The following attributes are hard coded (can't be changed): `s|M:file|d|F` -* fileSplit, fileDestinationSplit, tableNameSplit: see :ref:`split-pdf-upload` +* `fileSplit`, `fileDestinationSplit`, `tableNameSplit`: see :ref:`split-pdf-upload` * Excel Import: QFQ offers functionality to directly import excel data into the database. This functionality can optionally be combined with saving the file by using the above parameters like `fileDestination`. @@ -2159,8 +2159,8 @@ file type. * [jpeg] - default: `-density 150 -quality 90` * *fileDestinationSplit* = `` - Target directory and filename pattern for the created & - split'ed files. Default .split/split... - If explicit given, respect that SVG needs a printf style for , whereas JPEG is numbered automatically. E.g. :: + split'ed files. Default .split/split... + If explicit given, respect that SVG needs a printf style for , whereas JPEG is numbered automatically. E.g. :: [svg] fileDestinationSplit = fileadmin/protected/{{id:R}}.{{filenameBase:V}}.%02d.svg [jpeg] fileDestinationSplit = fileadmin/protected/{{id:R}}.{{filenameBase:V}}.jpg diff --git a/Documentation/Links.rst b/Documentation/Links.rst deleted file mode 100644 index defbc6e8..00000000 --- a/Documentation/Links.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. ================================================== -.. ================================================== -.. ================================================== -.. Header hierarchy -.. == -.. -- -.. ^^ -.. "" -.. ;; -.. ,, -.. -.. --------------------------------------------used to the update the records specified ------ -.. Best Practice T3 reST: https://docs.typo3.org/m/typo3/docs-how-to-document/master/en-us/WritingReST/CheatSheet.html -.. Reference: https://docs.typo3.org/m/typo3/docs-how-to-document/master/en-us/WritingReST/Index.html -.. Italic *italic* -.. Bold **bold** -.. Code ``text`` -.. External Links: `Bootstrap `_ -.. Add Images: .. image:: ../Images/a4.jpg -.. -.. -.. Admonitions -.. .. note:: .. important:: .. tip:: .. warning:: -.. Color: (blue) (orange) (green) (red) -.. -.. Definition: -.. some text becomes strong (only one line) -.. description has to indented - -.. -*- coding: utf-8 -*- with BOM. - -.. include:: Includes.txt - - -.. _links: - -Links ------ - -The links to issue and the GitHub repository are maintained in the Settings.cfg. - -You may want to remove this file if all important links are already handled in -Settings.cfg. - -:Packagist: - https://packagist.org/packages// - -:TER: - https://typo3.org/extensions/repository/view/ - -:Issues: - https://github.com///issues - -:GitHub Repository: - https://github.com// - -:Contact: - `@ `__ diff --git a/Documentation/index.rst b/Documentation/index.rst index be57bdf8..643d134e 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -87,7 +87,6 @@ This documentation is for the TYPO3 extension **qfq**. ApplicationTest GeneralTips Release - Links License Sitemap SearchDocs -- GitLab From cddec2f9d51bffb4e0cf5d32b2e69bc2b55eb0ea Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Sun, 21 Jun 2020 21:16:50 +0200 Subject: [PATCH 5/9] Refs #10778: First implementation of Upload & Unzip --- extension/Classes/Core/Constants.php | 4 +- extension/Classes/Core/Form/FormAction.php | 53 +----- extension/Classes/Core/Helper/HelperFile.php | 40 +++++ .../Classes/Core/Helper/HelperFormElement.php | 47 +++++ extension/Classes/Core/Save.php | 166 ++++++++++++++---- 5 files changed, 225 insertions(+), 85 deletions(-) diff --git a/extension/Classes/Core/Constants.php b/extension/Classes/Core/Constants.php index 3e03467c..2f39901e 100644 --- a/extension/Classes/Core/Constants.php +++ b/extension/Classes/Core/Constants.php @@ -242,7 +242,7 @@ const ERROR_STORE_KEY_EXIST = 1201; // I/O Error const ERROR_IO_COPY = 1300; - +const ERROR_IO_ZIP_OPEN = 1301; const ERROR_IO_RMDIR = 1302; const ERROR_IO_WRITE = 1303; const ERROR_IO_OPEN = 1304; @@ -1129,6 +1129,8 @@ const FE_FILE_REPLACE_MODE = 'fileReplace'; // Flag if a) QFQ throw an error if const FE_FILE_REPLACE_MODE_ALWAYS = 'always'; // Value for flag FE_FILE_REPLACE_MODE const FE_FILE_MIME_TYPE_ACCEPT = 'accept'; // Comma separated list of mime types const FE_FILE_MAX_FILE_SIZE = SYSTEM_FILE_MAX_FILE_SIZE; // Max upload file size +const FE_FILE_UNZIP = 'fileUnzip'; // 0|1|dir|{{SELECT ...}} +const FE_FILE_UNPACK_DIR = 'unpack'; // default dir if not specified const FE_FILE_CAPTURE = 'capture'; // On a smartphone opens the camera const FE_FILE_SPLIT = 'fileSplit'; diff --git a/extension/Classes/Core/Form/FormAction.php b/extension/Classes/Core/Form/FormAction.php index c452db09..aed3b24c 100644 --- a/extension/Classes/Core/Form/FormAction.php +++ b/extension/Classes/Core/Form/FormAction.php @@ -200,7 +200,7 @@ class FormAction { $this->store->setStore($arr, STORE_LDAP, true); } - $this->sqlValidate($fe); + HelperFormElement::sqlValidate($this->evaluate, $fe); if ($fe[FE_TYPE] === FE_TYPE_SENDMAIL) { $this->doSendMail($fe); @@ -291,57 +291,6 @@ class FormAction { $sendMail->process($mailConfig); } - /** - * If there is a query defined in fe.parameter.FE_SQL_VALIDATE: fire them. - * Count the selected records and compare them with fe.parameter.FE_EXPECT_RECORDS. - * If match: everything is fine, do nothing. - * Else throw \UserFormException with error message of fe.parameter.FE_MESSAGE_FAIL - * - * @param array $fe - * - * @throws \CodeException - * @throws \DbException - * @throws \UserFormException - * @throws \UserReportException - */ - private function sqlValidate(array $fe) { - - // Is there something to check? - if ($fe[FE_SQL_VALIDATE] === '') { - return; - } - - if ($fe[FE_EXPECT_RECORDS] === '') { - throw new \UserFormException("Missing parameter '" . FE_EXPECT_RECORDS . "'", ERROR_MISSING_EXPECT_RECORDS); - } - $expect = $this->evaluate->parse($fe[FE_EXPECT_RECORDS]); - - if ($fe[FE_MESSAGE_FAIL] === '') { - throw new \UserFormException("Missing parameter '" . FE_MESSAGE_FAIL . "'", ERROR_MISSING_MESSAGE_FAIL); - } - - // Do the check - $result = $this->evaluate->parse($fe[FE_SQL_VALIDATE], ROW_REGULAR); - if (!is_array($result)) { - throw new \UserFormException("Expected an array for '" . FE_SQL_VALIDATE . "', got a scalar. Please check for {{!...", ERROR_EXPECTED_ARRAY); - } - - // If there is at least one record count given, who matches: return 'check succeeded' - $countRecordsArr = explode(',', $expect); - foreach ($countRecordsArr as $count) { - if (count($result) == $count) { - return; // check successfully passed - } - } - - $msg = $this->evaluate->parse($fe[FE_MESSAGE_FAIL]); // Replace possible dynamic parts - - // Throw user error message - throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => $msg - , ERROR_MESSAGE_TO_DEVELOPER => 'validate() failed']), ERROR_REPORT_FAILED_ACTION); - - } - /** * Process slaveId, sqlBefore, sqlInsert|sqlUpdate|sqlDelete, sqlAfter. * flagFeAction=false: for Native Elements diff --git a/extension/Classes/Core/Helper/HelperFile.php b/extension/Classes/Core/Helper/HelperFile.php index 5619f9c0..a9b19352 100644 --- a/extension/Classes/Core/Helper/HelperFile.php +++ b/extension/Classes/Core/Helper/HelperFile.php @@ -101,6 +101,8 @@ class HelperFile { /** * Returns an array with filestat information to $pathFileName + * - mimeType + * - fileSize * * @param $pathFileName * @return array @@ -540,5 +542,43 @@ class HelperFile { return $pre . $separator . $post; } + + /** + * Translates ZIP error codes to text. + * + * @param $errno + * @return string + */ + public static function zipFileErrMsg($errno) { + + // using constant name as a string to make this function PHP4 compatible + $zipFileFunctionsErrors = array( + 'ZIPARCHIVE::ER_MULTIDISK' => 'Multi-disk zip archives not supported.', + 'ZIPARCHIVE::ER_RENAME' => 'Renaming temporary file failed.', + 'ZIPARCHIVE::ER_CLOSE' => 'Closing zip archive failed', + 'ZIPARCHIVE::ER_SEEK' => 'Seek error', + 'ZIPARCHIVE::ER_READ' => 'Read error', + 'ZIPARCHIVE::ER_WRITE' => 'Write error', + 'ZIPARCHIVE::ER_CRC' => 'CRC error', + 'ZIPARCHIVE::ER_ZIPCLOSED' => 'Containing zip archive was closed', + 'ZIPARCHIVE::ER_NOENT' => 'No such file.', + 'ZIPARCHIVE::ER_EXISTS' => 'File already exists', + 'ZIPARCHIVE::ER_OPEN' => 'Can\'t open file', + 'ZIPARCHIVE::ER_TMPOPEN' => 'Failure to create temporary file.', + 'ZIPARCHIVE::ER_ZLIB' => 'Zlib error', + 'ZIPARCHIVE::ER_MEMORY' => 'Memory allocation failure', + 'ZIPARCHIVE::ER_CHANGED' => 'Entry has been changed', + 'ZIPARCHIVE::ER_COMPNOTSUPP' => 'Compression method not supported.', + 'ZIPARCHIVE::ER_EOF' => 'Premature EOF', + 'ZIPARCHIVE::ER_INVAL' => 'Invalid argument', + 'ZIPARCHIVE::ER_NOZIP' => 'Not a zip archive', + 'ZIPARCHIVE::ER_INTERNAL' => 'Internal error', + 'ZIPARCHIVE::ER_INCONS' => 'Zip archive inconsistent', + 'ZIPARCHIVE::ER_REMOVE' => 'Can\'t remove file', + 'ZIPARCHIVE::ER_DELETED' => 'Entry has been deleted', + ); + + return $zipFileFunctionsErrors[$errno] ?? 'unknown'; + } } diff --git a/extension/Classes/Core/Helper/HelperFormElement.php b/extension/Classes/Core/Helper/HelperFormElement.php index 33c8dcd1..cb467d8f 100644 --- a/extension/Classes/Core/Helper/HelperFormElement.php +++ b/extension/Classes/Core/Helper/HelperFormElement.php @@ -9,6 +9,7 @@ namespace IMATHUZH\Qfq\Core\Helper; use IMATHUZH\Qfq\Core\Store\Store; +use IMATHUZH\Qfq\Core\Evaluate; /** @@ -861,5 +862,51 @@ EOF; return ''; } + /** + * If there is a query defined in fe.parameter.FE_SQL_VALIDATE: fire them. + * Count the selected records and compare them with fe.parameter.FE_EXPECT_RECORDS. + * If match: everything is fine, do nothing. + * Else throw \UserFormException with error message of fe.parameter.FE_MESSAGE_FAIL + * + * @param array $fe + * @param Evaluate $evaluate + * @throws \UserFormException + */ + public static function sqlValidate(Evaluate $evaluate, array $fe) { + // Is there something to check? + if ($fe[FE_SQL_VALIDATE] === '') { + return; + } + + if ($fe[FE_EXPECT_RECORDS] === '') { + throw new \UserFormException("Missing parameter '" . FE_EXPECT_RECORDS . "'", ERROR_MISSING_EXPECT_RECORDS); + } + $expect = $evaluate->parse($fe[FE_EXPECT_RECORDS]); + + if ($fe[FE_MESSAGE_FAIL] === '') { + throw new \UserFormException("Missing parameter '" . FE_MESSAGE_FAIL . "'", ERROR_MISSING_MESSAGE_FAIL); + } + + // Do the check + $result = $evaluate->parse($fe[FE_SQL_VALIDATE], ROW_REGULAR); + if (!is_array($result)) { + throw new \UserFormException("Expected an array for '" . FE_SQL_VALIDATE . "', got a scalar. Please check for {{!...", ERROR_EXPECTED_ARRAY); + } + + // If there is at least one record count given, who matches: return 'check succeeded' + $countRecordsArr = explode(',', $expect); + foreach ($countRecordsArr as $count) { + if (count($result) == $count) { + return; // check successfully passed + } + } + + $msg = $evaluate->parse($fe[FE_MESSAGE_FAIL]); // Replace possible dynamic parts + + // Throw user error message + throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => $msg + , ERROR_MESSAGE_TO_DEVELOPER => 'validate() failed']), ERROR_REPORT_FAILED_ACTION); + + } } \ No newline at end of file diff --git a/extension/Classes/Core/Save.php b/extension/Classes/Core/Save.php index 886444ca..5f98928f 100644 --- a/extension/Classes/Core/Save.php +++ b/extension/Classes/Core/Save.php @@ -20,6 +20,7 @@ use IMATHUZH\Qfq\Core\Helper\Support; use IMATHUZH\Qfq\Core\Store\FillStoreForm; use IMATHUZH\Qfq\Core\Store\Sip; use IMATHUZH\Qfq\Core\Store\Store; +use ZipArchive; /** * Class Save @@ -254,7 +255,7 @@ class Save { $formValues = $this->createEmptyTemplateGroupElements($formValues); // Iterate over all table.columns. Built an assoc array $newValues. - foreach ($tableColumns AS $column) { + foreach ($tableColumns as $column) { // Never save a predefined 'id': autoincrement values will be given by database.. if ($column === COLUMN_ID) { @@ -408,7 +409,7 @@ class Save { */ private function isColumnUploadField($feName) { - foreach ($this->feSpecNative AS $formElement) { + foreach ($this->feSpecNative as $formElement) { if ($formElement[FE_NAME] === $feName && $formElement[FE_TYPE] == FE_TYPE_UPLOAD) return true; } @@ -501,12 +502,22 @@ class Save { $sip = new Sip(false); $newValues = array(); - $vars = array(); + + $flagDoUnzip = false; $formValues = $this->store->getStore(STORE_FORM); $primaryRecord = $this->store->getStore(STORE_RECORD); // necessary to check if the current formElement exist as a column of the primary table. - foreach ($this->feSpecNative AS $formElement) { + // Upload - Take care the necessary target directories exist. + $cwd = getcwd(); + $sitePath = $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM); + if ($cwd === false || $sitePath === false || !HelperFile::chdir($sitePath)) { + throw new \UserFormException( + json_encode([ERROR_MESSAGE_TO_USER => 'getcwd() failed or SITE_PATH undefined or chdir() failed', ERROR_MESSAGE_TO_DEVELOPER => "getcwd() failed or SITE_PATH undefined or chdir('$sitePath') failed."]), + ERROR_IO_CHDIR); + } + + foreach ($this->feSpecNative as $formElement) { // skip non upload formElements if ($formElement[FE_TYPE] != FE_TYPE_UPLOAD) { continue; @@ -523,7 +534,32 @@ class Save { } $column = $formElement[FE_NAME]; + + $statusUpload = $this->store->getVar($formValues[$column] ?? '', STORE_EXTRA); + // Get file stats + $vars = array(); + $vars[VAR_FILE_SIZE] = $statusUpload[FILES_SIZE]; + $vars[VAR_FILE_MIME_TYPE] = $statusUpload[FILES_TYPE]; + + // Check for 'unzip'. + if (isset($formElement[FE_FILE_UNZIP]) && $formElement[FE_FILE_UNZIP] != '0' && $vars[VAR_FILE_MIME_TYPE] = 'application/zip') { + $flagDoUnzip = true; + } + + // Do upload $pathFileName = $this->doUpload($formElement, ($formValues[$column] ?? ''), $sip, $modeUpload); + if ($flagDoUnzip) { + if ($formElement[FE_FILE_UNZIP] == '' || $formElement[FE_FILE_UNZIP] == '1') { + // Set default dir. + $formElement[FE_FILE_UNZIP] = HelperFile::joinPathFilename(dirname($pathFileName), FE_FILE_UNPACK_DIR); + } + + // Backup STORE_VAR - will be changed in doUnzip() + $tmpStoreVar = $this->store->getStore(STORE_VAR); + $this->doUnzip($formElement, $pathFileName); + // Restore STORE_VAR + $this->store->setStore($tmpStoreVar, STORE_VAR, true); + } if ($modeUpload == UPLOAD_MODE_DELETEOLD && $pathFileName == '') { $pathFileNameTmp = ''; // see '4' @@ -540,15 +576,15 @@ class Save { // No new upload and no existing: take care to remove previous upload file statistics. $this->store->unsetVar(VAR_FILE_MIME_TYPE, STORE_VAR); $this->store->unsetVar(VAR_FILE_SIZE, STORE_VAR); - $vars[VAR_FILE_SIZE] = 0; - $vars[VAR_FILE_MIME_TYPE] = ''; } else { - $vars = HelperFile::getFileStat($pathFileNameTmp); + $this->store->appendToStore($vars, STORE_VAR); } // If given: fire a sqlBefore query - $this->evaluate->parse($formElement[FE_SQL_BEFORE]); + if (!$flagDoUnzip) { + $this->evaluate->parse($formElement[FE_SQL_BEFORE]); + } // Upload Type: Simple or Advanced // If (isset($primaryRecord[$column])) { - see #5048 - isset does not deal correctly with NULL! @@ -567,22 +603,97 @@ class Save { } } elseif (isset($formElement[FE_IMPORT_TO_TABLE]) && !isset($formElement[FE_SLAVE_ID])) { // Excel import on nonexisting column -> no upload + } elseif ($flagDoUnzip) { + // If ZIP and advanced upload: process it not here but via doUnzip. } else { // 'Advanced Upload' $this->doUploadSlave($formElement, $modeUpload); } // If given: fire a sqlAfter query - $this->evaluate->parse($formElement[FE_SQL_AFTER]); - + if (!$flagDoUnzip) { + $this->evaluate->parse($formElement[FE_SQL_AFTER]); + } } + // Clean up + HelperFile::chdir($cwd); + // Only used in 'Simple Upload' if (count($newValues) > 0) { $this->updateRecord($this->formSpec[F_TABLE_NAME], $newValues, $recordId, $this->formSpec[F_PRIMARY_KEY]); } } + /** + * Unzip $pathFileName to $formElement[FE_FILE_UNZIP]. Before final extract, fire FE_SQL_VALIDATE. + * For each file in ZIP: + * - Fill STORE_VAR with VAR_FILENAME, VAR_FILENAME_ONLY, VAR_FILENAME_BASE, VAR_FILENAME_EXT, VAR_FILE_MIME_TYPE, VAR_FILE_SIZE. + * - Fire $formElement[FE_SQL_VALIDATE] + * - Fire FE_SLAVE_ID, FE_SQL_BEFORE, FE_SQL_INSERT, FE_SQL_UPDATE, FE_SQL_DELETE, FE_SQL_AFTER + * + * @param array $formElement + * @param string $pathFileName + * @throws \CodeException + * @throws \DbException + * @throws \UserFormException + * @throws \UserReportException + */ + private function doUnzip(array $formElement, $pathFileName) { + + if (!is_readable($pathFileName)) { + throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => "Open ZIP file failed", + ERROR_MESSAGE_TO_DEVELOPER => "File: " . $pathFileName]), + ERROR_IO_ZIP_OPEN); + } + + $zip = new ZipArchive(); + $res = $zip->open($pathFileName); + if ($res !== true) { + throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => "Open ZIP file failed" . HelperFile::zipFileErrMsg($res), + ERROR_MESSAGE_TO_DEVELOPER => "File: " . $pathFileName]), ERROR_IO_ZIP_OPEN); + } + + // Do sqlValidate() before final extraction. + if (!empty($formElement[FE_SQL_VALIDATE])) { + for ($i = 0; $i < $zip->numFiles; $i++) { + $stat = $zip->statIndex($i); + + $itemPathFileName = HelperFile::joinPathFilename($formElement[FE_FILE_UNZIP], $stat['name']); + $this->store->appendToStore(HelperFile::getFileStat($itemPathFileName), STORE_VAR); + $this->store->appendToStore(HelperFile::pathinfo($itemPathFileName), STORE_VAR); + + HelperFormElement::sqlValidate($this->evaluate, $formElement); + } + } + + // Extract + if (false === $zip->extractTo($formElement[FE_FILE_UNZIP])) { + throw new \UserFormException("Failed to extract ZIP.", ERROR_IO_ZIP_OPEN); + } + // Close Zip + if (false === $zip->close()) { + throw new \UserFormException("Failed to close ZIP.", ERROR_IO_ZIP_OPEN); + } + + // Process + if (!empty($formElement[FE_SLAVE_ID] . $formElement[FE_SQL_BEFORE] . $formElement[FE_SQL_INSERT] . + $formElement[FE_SQL_UPDATE] . $formElement[FE_SQL_DELETE] . $formElement[FE_SQL_AFTER])) { + for ($i = 0; $i < $zip->numFiles; $i++) { + $stat = $zip->statIndex($i); + + $itemPathFileName = HelperFile::joinPathFilename($formElement[FE_FILE_UNZIP], $stat['name']); + $this->store->appendToStore(HelperFile::getFileStat($itemPathFileName), STORE_VAR); + $this->store->appendToStore(HelperFile::pathinfo($itemPathFileName), STORE_VAR); + + $this->evaluate->parse($formElement[FE_SQL_BEFORE]); + $this->doUploadSlave($formElement, UPLOAD_MODE_NEW); + $this->evaluate->parse($formElement[FE_SQL_AFTER]); + print_r(basename($stat['name']) . PHP_EOL); + } + } + } + /** * Process all Upload FormElements for the given $recordId. * After processing, &$formValues will be updated with the final filename. @@ -592,7 +703,7 @@ class Save { */ public function processAllImageCutFE() { - foreach ($this->feSpecNative AS $formElement) { + foreach ($this->feSpecNative as $formElement) { // skip non upload formElements if ($formElement[FE_TYPE] != FE_TYPE_IMAGE_CUT) { continue; @@ -631,7 +742,7 @@ class Save { $flagAllRequiredGiven = 1; - foreach ($this->feSpecNative AS $key => $formElement) { + foreach ($this->feSpecNative as $key => $formElement) { // Do not check retype slave FE. if (isset($formElement[FE_RETYPE_SOURCE_NAME])) { @@ -748,26 +859,26 @@ class Save { * Process upload for the given Formelement. If necessary, delete a previous uploaded file. * Calculate the final path/filename and move the file to the new location. * - * Check also: doc/CODING.md + * Check also: Documentation-develop/CODING.md * * @param array $formElement FormElement 'upload' * @param string $sipUpload SIP * @param Sip $sip * @param string $modeUpload UPLOAD_MODE_UNCHANGED | UPLOAD_MODE_NEW | UPLOAD_MODE_DELETEOLD | * UPLOAD_MODE_DELETEOLD_NEW - * * @return false|string New pathFilename or false on error * @throws \CodeException * @throws \DbException - * @throws \UserFormException - * @throws \UserReportException * @throws \PhpOffice\PhpSpreadsheet\Exception * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception + * @throws \UserFormException + * @throws \UserReportException * @internal param $recordId */ private function doUpload($formElement, $sipUpload, Sip $sip, &$modeUpload) { $flagDelete = false; $modeUpload = UPLOAD_MODE_UNCHANGED; + $pathFileName = ''; // Status information about upload file $statusUpload = $this->store->getVar($sipUpload, STORE_EXTRA); @@ -781,15 +892,6 @@ class Save { $this->doImport($formElement, $tmpFile); } - // Upload - Take care the necessary target directories exist. - $cwd = getcwd(); - $sitePath = $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM); - if ($cwd === false || $sitePath === false || !HelperFile::chdir($sitePath)) { - throw new \UserFormException( - json_encode([ERROR_MESSAGE_TO_USER => 'getcwd() failed or SITE_PATH undefined or chdir() failed', ERROR_MESSAGE_TO_DEVELOPER => "getcwd() failed or SITE_PATH undefined or chdir('$sitePath') failed."]), - ERROR_IO_CHDIR); - } - // Delete existing old file. if (isset($statusUpload[FILES_FLAG_DELETE]) && $statusUpload[FILES_FLAG_DELETE] == '1') { $arr = $sip->getVarsFromSip($sipUpload); @@ -819,8 +921,6 @@ class Save { Logger::logMessageWithPrefix($msg, $this->qfqLogFilename); } - HelperFile::chdir($cwd); - // Delete current used uniq SIP $this->store->setVar($sipUpload, array(), STORE_EXTRA); @@ -828,6 +928,8 @@ class Save { } /** + * Excel Import + * * @param $formElement * @param $fileName * @throws \CodeException @@ -977,7 +1079,7 @@ class Save { } // Import the data - foreach ($worksheetData AS $rowIndex => $row) { + foreach ($worksheetData as $rowIndex => $row) { $columnList = '`' . implode('`,`', $columnListArr) . '`'; $paramPlaceholders = str_repeat('?,', count($worksheetData[0]) - 1) . '?'; $insertSql = "INSERT INTO `$tableName` ($columnList) VALUES ($paramPlaceholders)"; @@ -989,7 +1091,7 @@ class Save { /** * Copy uploaded file from temporary location to final location. * - * Check also: doc/CODING.md + * Check also: Documentation-develop/CODING.md * * @param array $formElement * @param array $statusUpload @@ -1087,9 +1189,9 @@ class Save { } /** - * Check's if the file $pathFileName should be splitted in one file per page. If no: do nothing and return. + * Check's if the file $pathFileName should be split'ed in one file per PDF page. If no: do nothing and return. * The only possible split target file format is 'svg': fileSplit=svg. - * The splitted files will be saved under fileDestinationSplit=some/path/to/file.%02d.svg. A printf style token, + * The split'ed files will be saved under fileDestinationSplit=some/path/to/file.%02d.svg. A printf style token, * like '%02d', is needed to create distinguished filename's. See 'man pdf2svg' for further details. * For every created file, a record in table 'Split' is created (see splitSvg() ), storing the pathFileName of the * current page/file. @@ -1221,7 +1323,7 @@ class Save { * Create/update or delete the slave record. * * @param array $fe - * @param $modeUpload + * @param string $modeUpload UPLOAD_MODE_NEW|UPLOAD_MODE_DELETEOLD_NEW|UPLOAD_MODE_DELETEOLD * @return int * @throws \CodeException * @throws \DbException -- GitLab From bcab137514c888a29ac4b8a796fabc7a19dcba57 Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Sun, 21 Jun 2020 23:16:53 +0200 Subject: [PATCH 6/9] Refs #10778: Fix problem with missing mimetype. --- Documentation-develop/CODING.md | 2 +- .../Classes/Core/Helper/HelperFormElement.php | 14 ++++++++----- extension/Classes/Core/Save.php | 21 ++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Documentation-develop/CODING.md b/Documentation-develop/CODING.md index b763de81..690582a0 100644 --- a/Documentation-develop/CODING.md +++ b/Documentation-develop/CODING.md @@ -162,7 +162,7 @@ Upload to server, before 'save' ............................... * If a user open's a file for upload via the browse button, that file is immediately transmitted to the server. The user will see a turning wheel until the upload finished. -* After successfull upload the 'Browse' button disappears and the filename, plus the delete button, will be displayed (client logic). +* After successfully upload the 'Browse' button disappears and the filename, plus the delete button, will be displayed (client logic). * The uploaded file will be checked: maxsize, mime type, check script. * The uploaded file is still temporary. It has been renamed from '[STORE_EXTRA][][FILES_TMP_NAME]' to '[STORE_EXTRA][][FILES_TMP_NAME].cached'. diff --git a/extension/Classes/Core/Helper/HelperFormElement.php b/extension/Classes/Core/Helper/HelperFormElement.php index cb467d8f..b969ac40 100644 --- a/extension/Classes/Core/Helper/HelperFormElement.php +++ b/extension/Classes/Core/Helper/HelperFormElement.php @@ -8,8 +8,8 @@ namespace IMATHUZH\Qfq\Core\Helper; -use IMATHUZH\Qfq\Core\Store\Store; use IMATHUZH\Qfq\Core\Evaluate; +use IMATHUZH\Qfq\Core\Store\Store; /** @@ -38,7 +38,7 @@ class HelperFormElement { */ public static function explodeParameterInArrayElements(array &$elements, $keyName) { - foreach ($elements AS $key => $element) { + foreach ($elements as $key => $element) { self::explodeParameter($element, $keyName); $elements[$key] = $element; } @@ -59,7 +59,7 @@ class HelperFormElement { // Do not add FE_SLAVE_ID - it's necessary to detect if a value is given or not. $default = [FE_SQL_BEFORE => '', FE_SQL_INSERT => '', FE_SQL_UPDATE => '', FE_SQL_DELETE => '', FE_SQL_AFTER => '']; - foreach ($elements AS $key => $element) { + foreach ($elements as $key => $element) { $elements[$key][FE_TG_INDEX] = 0; unset($elements[$key][FE_ADMIN_NOTE]); // $elements[$key][FE_DATA_REFERENCE] = ''; @@ -92,7 +92,7 @@ class HelperFormElement { if (!$flagAllowOverwrite) { // Check if some of the exploded keys conflict with existing keys $checkKeys = array_keys($arr); - foreach ($checkKeys AS $checkKey) { + foreach ($checkKeys as $checkKey) { if (!empty($element[$checkKey])) { self::$store = Store::getInstance(); self::$store->setVar(SYSTEM_FORM_ELEMENT, Logger::formatFormElementName($element), STORE_SYSTEM); @@ -870,7 +870,10 @@ EOF; * * @param array $fe * @param Evaluate $evaluate + * @throws \CodeException + * @throws \DbException * @throws \UserFormException + * @throws \UserReportException */ public static function sqlValidate(Evaluate $evaluate, array $fe) { @@ -906,7 +909,8 @@ EOF; // Throw user error message throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => $msg - , ERROR_MESSAGE_TO_DEVELOPER => 'validate() failed']), ERROR_REPORT_FAILED_ACTION); + , ERROR_MESSAGE_TO_DEVELOPER => "validate() failed.\nSQL Raw: " . $fe[FE_SQL_VALIDATE]]) + , ERROR_REPORT_FAILED_ACTION); } } \ No newline at end of file diff --git a/extension/Classes/Core/Save.php b/extension/Classes/Core/Save.php index 5f98928f..2bb38b04 100644 --- a/extension/Classes/Core/Save.php +++ b/extension/Classes/Core/Save.php @@ -538,8 +538,8 @@ class Save { $statusUpload = $this->store->getVar($formValues[$column] ?? '', STORE_EXTRA); // Get file stats $vars = array(); - $vars[VAR_FILE_SIZE] = $statusUpload[FILES_SIZE]; - $vars[VAR_FILE_MIME_TYPE] = $statusUpload[FILES_TYPE]; + $vars[VAR_FILE_SIZE] = $statusUpload[FILES_SIZE] ?? ''; + $vars[VAR_FILE_MIME_TYPE] = $statusUpload[FILES_TYPE] ?? ''; // Check for 'unzip'. if (isset($formElement[FE_FILE_UNZIP]) && $formElement[FE_FILE_UNZIP] != '0' && $vars[VAR_FILE_MIME_TYPE] = 'application/zip') { @@ -548,7 +548,7 @@ class Save { // Do upload $pathFileName = $this->doUpload($formElement, ($formValues[$column] ?? ''), $sip, $modeUpload); - if ($flagDoUnzip) { + if ($flagDoUnzip && $pathFileName != '') { if ($formElement[FE_FILE_UNZIP] == '' || $formElement[FE_FILE_UNZIP] == '1') { // Set default dir. $formElement[FE_FILE_UNZIP] = HelperFile::joinPathFilename(dirname($pathFileName), FE_FILE_UNPACK_DIR); @@ -654,7 +654,12 @@ class Save { ERROR_MESSAGE_TO_DEVELOPER => "File: " . $pathFileName]), ERROR_IO_ZIP_OPEN); } - // Do sqlValidate() before final extraction. + // Extract + if (false === $zip->extractTo($formElement[FE_FILE_UNZIP])) { + throw new \UserFormException("Failed to extract ZIP.", ERROR_IO_ZIP_OPEN); + } + + // Do sqlValidate() - to get mime type of zipped items, the archive has already been extracted. if (!empty($formElement[FE_SQL_VALIDATE])) { for ($i = 0; $i < $zip->numFiles; $i++) { $stat = $zip->statIndex($i); @@ -667,16 +672,16 @@ class Save { } } - // Extract - if (false === $zip->extractTo($formElement[FE_FILE_UNZIP])) { - throw new \UserFormException("Failed to extract ZIP.", ERROR_IO_ZIP_OPEN); - } // Close Zip if (false === $zip->close()) { throw new \UserFormException("Failed to close ZIP.", ERROR_IO_ZIP_OPEN); } // Process + if (!isset($formElement[FE_SLAVE_ID])) { + $formElement[FE_SLAVE_ID] = ''; + } + if (!empty($formElement[FE_SLAVE_ID] . $formElement[FE_SQL_BEFORE] . $formElement[FE_SQL_INSERT] . $formElement[FE_SQL_UPDATE] . $formElement[FE_SQL_DELETE] . $formElement[FE_SQL_AFTER])) { for ($i = 0; $i < $zip->numFiles; $i++) { -- GitLab From 06eceeb03283ca8fa7062ddad684c5f40b25f1fe Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Mon, 22 Jun 2020 00:39:22 +0200 Subject: [PATCH 7/9] Refs #10778: Fix problem closing ZIP too early. --- extension/Classes/Core/Save.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extension/Classes/Core/Save.php b/extension/Classes/Core/Save.php index 2bb38b04..ca3a758e 100644 --- a/extension/Classes/Core/Save.php +++ b/extension/Classes/Core/Save.php @@ -672,11 +672,6 @@ class Save { } } - // Close Zip - if (false === $zip->close()) { - throw new \UserFormException("Failed to close ZIP.", ERROR_IO_ZIP_OPEN); - } - // Process if (!isset($formElement[FE_SLAVE_ID])) { $formElement[FE_SLAVE_ID] = ''; @@ -694,9 +689,13 @@ class Save { $this->evaluate->parse($formElement[FE_SQL_BEFORE]); $this->doUploadSlave($formElement, UPLOAD_MODE_NEW); $this->evaluate->parse($formElement[FE_SQL_AFTER]); - print_r(basename($stat['name']) . PHP_EOL); } } + + // Close Zip + if (false === $zip->close()) { + throw new \UserFormException("Failed to close ZIP.", ERROR_IO_ZIP_OPEN); + } } /** -- GitLab From d18b69d19e76b6c48b807bec98b0ffc5d2f58694 Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Mon, 22 Jun 2020 15:18:07 +0200 Subject: [PATCH 8/9] Refs #10778: Fix problem detecting ZIP on upload --- extension/Classes/Core/Save.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extension/Classes/Core/Save.php b/extension/Classes/Core/Save.php index ca3a758e..5e9cca3c 100644 --- a/extension/Classes/Core/Save.php +++ b/extension/Classes/Core/Save.php @@ -542,7 +542,9 @@ class Save { $vars[VAR_FILE_MIME_TYPE] = $statusUpload[FILES_TYPE] ?? ''; // Check for 'unzip'. - if (isset($formElement[FE_FILE_UNZIP]) && $formElement[FE_FILE_UNZIP] != '0' && $vars[VAR_FILE_MIME_TYPE] = 'application/zip') { + if (isset($formElement[FE_FILE_UNZIP]) + && $formElement[FE_FILE_UNZIP] != '0' + && $vars[VAR_FILE_MIME_TYPE] == 'application/zip') { $flagDoUnzip = true; } -- GitLab From fa1a8d01db382c5b97375db008f837ee2579dd88 Mon Sep 17 00:00:00 2001 From: Carsten Rose Date: Wed, 24 Jun 2020 13:13:08 +0200 Subject: [PATCH 9/9] Manual.rst: add doc for F10778 --- Documentation/Form.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Documentation/Form.rst b/Documentation/Form.rst index 6157a9d3..1c44bdfa 100644 --- a/Documentation/Form.rst +++ b/Documentation/Form.rst @@ -2027,6 +2027,23 @@ FormElement.parameter * The following attributes are hard coded (can't be changed): `s|M:file|d|F` +* *fileUnzip* - If the file is a ZIP file (only then) it will be unzipped. If no directory is given via ``fileUnzip``, the + basedir of ``fileDestination`` is taken, appended by ``unpack``. + + If an unzip will be done, for each file of the archive STORE_VAR will be filled (name, path of the extracted file, + mime type, size) and the following will be triggered: *sqlValidate, slaveId, sqlBefore, sqlAfter, sqlInsert, sqlUpdate*. + + Example:: + + fileDestination = fileadmin/file_{{id:R}}.zip + fileUnzip + sqlValidate ={{! SELECT '' FROM (SELECT '') AS fake WHERE '{{mimeType:V}}' LIKE 'application/pdf%' }} + expectRecords=1 + messageFail=Unexpected filetype + + # Set new + sqlAfter={{INSERT INTO Upload (pathFileName) VALUES '{{filename:V}}' }} + * `fileSplit`, `fileDestinationSplit`, `tableNameSplit`: see :ref:`split-pdf-upload` * Excel Import: QFQ offers functionality to directly import excel data into the database. This functionality can -- GitLab