Merge branch 'B9512pdfMergeFails' into 'master'

B9512pdf merge fails

See merge request !191

Documentation/Manual.rst

@@ -51,6 +51,7 @@ The following features are only tested / supported on linux hosts:
 * General: QFQ is coded to run on Linux hosts, preferable on Debian derivates like Ubuntu.
 * HTML to PDF conversion - command `wkhtmltopdf`.
 * Concatenation of PDF files - command `pdfunite`.
+* PDF decrypt (used for merge with pdfunite) - command `qpdf`.
 * Mime type detection for uploads - command `file`.
 
 
@@ -66,12 +67,12 @@ To normalize UTF8 input, *php-intl* package is needed by
 
 * normalizer::normalize()
 
-For the `download`_ function, the programs `pdfunite` and `file` are necessary to concatenate PDF files.
+For the `download`_ function, the programs `pdfunite`, `qpdf` and `file` are necessary to concatenate PDF files.
 
 Preparation for Ubuntu::
 
   sudo apt install php-intl
-  sudo apt install poppler-utils libxrender1 file pdf2svg  # for file upload, PDF and 'HTML to PDF' (wkhtmltopdf), PDF split
+  sudo apt install poppler-utils libxrender1 file pdf2svg pdfunite qpdf # for file upload, PDF and 'HTML to PDF' (wkhtmltopdf), PDF split
   sudo apt install inkscape imagemagick            # to render thumbnails
 
 .. _wkhtml:

extension/Classes/Core/Helper/HelperFile.php

@@ -247,9 +247,9 @@ class HelperFile {
     public static function chmod($pathFileName, $mode = false) {
 
         if ($mode !== false) {
-            if (false === chmod($pathFileName, $mode)) {
+            if (false === @chmod($pathFileName, $mode)) {
                 throw new \UserFormException(
-                    json_encode([ERROR_MESSAGE_TO_USER => 'Failed: chmod', ERROR_MESSAGE_TO_DEVELOPER => "Failed: chmod $mode '$pathFileName'"]),
+                    json_encode([ERROR_MESSAGE_TO_USER => 'Failed: chmod', ERROR_MESSAGE_TO_DEVELOPER => self::errorGetLastAsString()]),
                     ERROR_IO_CHMOD);
             }
         }
@@ -277,7 +277,7 @@ class HelperFile {
      */
     public static function chdir($cwd) {
 
-        if (false === chdir($cwd)) {
+        if (false === @chdir($cwd)) {
             $msg = self::errorGetLastAsString() . " - chdir($cwd)";
             throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => 'chdir failed', ERROR_MESSAGE_TO_DEVELOPER => $msg]), ERROR_IO_CHDIR);
         }
@@ -300,7 +300,7 @@ class HelperFile {
             Logger::logMessageWithPrefix("Unlink: $filename", $logFilename);
         }
 
-        if (false === unlink($filename)) {
+        if (false === @unlink($filename)) {
             $msg = self::errorGetLastAsString() . " - unlink($filename)";
             throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => 'unlink failed', ERROR_MESSAGE_TO_DEVELOPER => $msg]), ERROR_IO_UNLINK);
         }
@@ -317,7 +317,7 @@ class HelperFile {
      */
     public static function rmdir($tempDir) {
 
-        if (false === rmdir($tempDir)) {
+        if (false === @rmdir($tempDir)) {
             $msg = self::errorGetLastAsString() . " - rmdir($tempDir)";
             throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => 'rmdir failed', ERROR_MESSAGE_TO_DEVELOPER => $msg]), ERROR_IO_RMDIR);
         }
@@ -335,7 +335,7 @@ class HelperFile {
      */
     public static function rename($oldname, $newname) {
 
-        if (false === rename($oldname, $newname)) {
+        if (false === @rename($oldname, $newname)) {
             $msg = self::errorGetLastAsString() . " - rename($oldname ,$newname)";
             throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => 'unlink failed', ERROR_MESSAGE_TO_DEVELOPER => $msg]), ERROR_IO_RENAME);
         }
@@ -358,7 +358,7 @@ class HelperFile {
             touch($dest);
         }
 
-        if (false === copy($source, $dest)) {
+        if (false === @copy($source, $dest)) {
 
             if (!is_readable($source)) {
                 throw new \UserFormException(json_encode([ERROR_MESSAGE_TO_USER => 'copy failed', ERROR_MESSAGE_TO_DEVELOPER => "Can't read file '$source'"]), ERROR_IO_READ_FILE);

extension/Classes/Core/Report/Download.php

@@ -10,20 +10,19 @@
 
 namespace IMATHUZH\Qfq\Core\Report;
 
+use IMATHUZH\Qfq\Core\Database\Database;
+use IMATHUZH\Qfq\Core\Helper\DownloadPage;
+use IMATHUZH\Qfq\Core\Helper\HelperFile;
 use IMATHUZH\Qfq\Core\Helper\KeyValueStringParser;
-use IMATHUZH\Qfq\Core\Store\Session;
-use IMATHUZH\Qfq\Core\Store\Store;
-use IMATHUZH\Qfq\Core\Store\Sip;
+use IMATHUZH\Qfq\Core\Helper\Logger;
 use IMATHUZH\Qfq\Core\Helper\OnArray;
 use IMATHUZH\Qfq\Core\Helper\OnString;
-use IMATHUZH\Qfq\Core\Helper\Logger;
 use IMATHUZH\Qfq\Core\Helper\Sanitize;
-use IMATHUZH\Qfq\Core\Helper\HelperFile;
-use IMATHUZH\Qfq\Core\Helper\DownloadPage;
-use IMATHUZH\Qfq\Core\QuickFormQuery;
-
-use IMATHUZH\Qfq\Core\Database\Database;
 use IMATHUZH\Qfq\Core\Helper\Support;
+use IMATHUZH\Qfq\Core\QuickFormQuery;
+use IMATHUZH\Qfq\Core\Store\Session;
+use IMATHUZH\Qfq\Core\Store\Sip;
+use IMATHUZH\Qfq\Core\Store\Store;
 
 /**
  * Class Download
@@ -147,7 +146,7 @@ class Download {
             Logger::logMessage("Download: $cmd", $this->downloadDebugLog);
         }
 
-        exec($cmd, $output, $rc);
+        $rc = $this->concatPdfFilesPdfUnite($cmd, $output);
 
         if ($rc != 0) {
             throw new \DownloadException (json_encode([ERROR_MESSAGE_TO_USER => "Failed to merge PDF file",
@@ -158,6 +157,69 @@ class Download {
         return $concatFile;
     }
 
+    /**
+     * Fires the merge command.
+     * If for any reason the command fails: check if the reason is 'unencrypted files'.
+     * If 'yes': try to decrypt them with qpdf.
+     * After one decrypt, try merge again.
+     * Try to merge and decrypt as long as there are encrypted files.
+     *
+     * @param $cmd
+     * @param $rcOutput
+     * @return mixed
+     * @throws \DownloadException
+     * @throws \UserFormException
+     */
+    private function concatPdfFilesPdfUnite($cmd, &$rcOutput) {
+        $last = '';
+        $rcOutput = '-';
+
+        // Try to merge the PDFs as long as a problematic PDF has been repaired.
+        while ($last != $rcOutput) {
+
+            $last = $rcOutput; // Remember last
+
+            // Merge
+            exec($cmd, $rcOutput, $rc);
+
+            if ($rc == 0) {
+                break; // skip rest if everything is fine
+            }
+
+            // Possible output: "Unimplemented Feature: Could not merge encrypted files ('ct.18.06.092-097.pdf')"
+            $line = implode(',', $rcOutput);
+            if (false !== strstr($line, "Unimplemented Feature: Could not merge encrypted files (")) {
+
+                $arr = explode("'", $line, 3);
+                if (!empty($arr[1]) && file_exists($arr[1])) {
+                    $file = $arr[1]; // problematic file
+
+                    // Create a backup file: only one per day!
+                    $backup = $file . date('.Y-m-d');
+                    if (!file_exists($backup)) {
+                        HelperFile::copy($file, $backup);
+                    }
+
+                    $cmdQpdf = "qpdf --decrypt '$backup' '$file' 2>&1"; // Try to decrypt file
+                    exec($cmdQpdf, $outputQpdf, $rcQpdf);
+
+                    if ($rcQpdf != 0) {
+                        // qpdf failed: restore origfile in case the $file has been destroyed.
+                        HelperFile::copy($backup, $file);
+                        throw new \DownloadException (json_encode([ERROR_MESSAGE_TO_USER => "Failed to decrypt PDF",
+                                ERROR_MESSAGE_TO_DEVELOPER => "CMD: " . $cmdQpdf . "<br>RC: $rc<br>Output: " . implode("<br>", $outputQpdf)])
+                            , ERROR_DOWNLOAD_MERGE_FAILED);
+                    }
+                }
+            } else {
+                throw new \DownloadException (json_encode([ERROR_MESSAGE_TO_USER => "Merge PDF file failed.",
+                        ERROR_MESSAGE_TO_DEVELOPER => "CMD: " . $cmd . "<br>RC: $rc<br>Output: " . implode("<br>", $rcOutput)])
+                    , ERROR_DOWNLOAD_MERGE_FAILED);
+            }
+        }
+        return $rc;
+    }
+
     /**
      * Get the mimetype of $filename and store them in $rcMimetype.
      * Checks if the extension of $outputFilename fit's to the mimetype. If not, append the mimetype extension.
@@ -168,7 +230,8 @@ class Download {
      *
      * @return string possible updated $outputFilename, according the mimetype.
      */
-    private function targetFilenameExtension($filename, $outputFilename, &$rcMimetype) {
+    private
+    function targetFilenameExtension($filename, $outputFilename, &$rcMimetype) {
 
         $rcMimetype = mime_content_type($filename);
 
@@ -182,7 +245,8 @@ class Download {
      * @param $outputFilename
      * @throws \DownloadException
      */
-    private function outputFile($file, $outputFilename) {
+    private
+    function outputFile($file, $outputFilename) {
 
         $json = '';
         $flagJson = ($this->getOutputFormat() === DOWNLOAD_OUTPUT_FORMAT_JSON);
@@ -237,7 +301,8 @@ class Download {
      * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
      * @throws \PhpOffice\PhpSpreadsheet\Writer\Exception
      */
-    private function getElement($element, $downloadMode, &$rcData) {
+    private
+    function getElement($element, $downloadMode, &$rcData) {
 
         $filename = '';
         $rcArgs = array();
@@ -323,7 +388,8 @@ class Download {
      * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
      * @throws \PhpOffice\PhpSpreadsheet\Writer\Exception
      */
-    private function getEvaluatedBodyText($uid, $urlParam) {
+    private
+    function getEvaluatedBodyText($uid, $urlParam) {
         foreach ($urlParam as $key => $paramValue) {
             $this->store->setVar($key, $paramValue, STORE_SIP);
         }
@@ -346,7 +412,8 @@ class Download {
      * @return string ZIP filename - has to be deleted later.
      * @throws \DownloadException
      */
-    private function zipFiles(array $files) {
+    private
+    function zipFiles(array $files) {
 
         $zipFile = HelperFile::tempnam();
         if (false === $zipFile) {
@@ -398,7 +465,8 @@ class Download {
      * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
      * @throws \PhpOffice\PhpSpreadsheet\Writer\Exception
      */
-    private function doElements(array $vars, $outputMode) {
+    private
+    function doElements(array $vars, $outputMode) {
 
         $tmpFiles = array();
 
@@ -519,7 +587,8 @@ class Download {
      * @throws \UserFormException
      * @throws \UserReportException
      */
-    private function doThumbnail($urlParam) {
+    private
+    function doThumbnail($urlParam) {
 
         $thumbnail = new Thumbnail();
         $pathFilenameThumbnail = $thumbnail->process($urlParam, THUMBNAIL_VIA_DOWNLOAD);
@@ -543,7 +612,8 @@ class Download {
      * @throws \PhpOffice\PhpSpreadsheet\Reader\Exception
      * @throws \PhpOffice\PhpSpreadsheet\Writer\Exception
      */
-    public function process($vars, $outputMode = OUTPUT_MODE_DIRECT) {
+    public
+    function process($vars, $outputMode = OUTPUT_MODE_DIRECT) {
 
         if (!is_array($vars)) {
             $vars = $this->store->getStore(STORE_SIP);
@@ -557,14 +627,16 @@ class Download {
     /**
      * @param $outputFormat
      */
-    private function setOutputFormat($outputFormat) {
+    private
+    function setOutputFormat($outputFormat) {
         $this->outputFormat = $outputFormat;
     }
 
     /**
      * @return string - DOWNLOAD_OUTPUT_FORMAT_RAW | DOWNLOAD_OUTPUT_FORMAT_JSON
      */
-    public function getOutputFormat() {
+    public
+    function getOutputFormat() {
         return $this->outputFormat;
     }
 }