Refs #12615. Implements silent HEIC/HEIF conversion to png. Write a note in...

Refs #12615. Implements silent HEIC/HEIF conversion to png. Write a note in realease notes, to install 'libheif-examples'.

Documentation-develop/CODING.md

@@ -178,6 +178,15 @@ Upload to server, before 'save'
   * An optional existing variable [STORE_EXTRA][<uploadSip>][FILES_TMP_NAME] will be deleted. The 'flagDelete' must not 
     be change - it's later needed to detect to delete, of earlier uploaded files.
 
+Upload HEIC/HEIF
+................
+As of 2021, latest iPhone & Samsung uses the HEIC or HEIF image format. These format isn't still fully supported under
+linux. Especially Ubuntu 18.04 has, without installed extra packages, no support at all. Ubuntu 20.04 knows the mimetype 
+but still need extra packages.
+
+As a temporary workaround, QFQ will silently convert uploaded HEIC/HEIF images to png. The package `libheif-examples`
+has to be installed. Detection of HEIC/HEIF images are done via `heif-info`, conversion via `heif-convert`.
+
 Form save
 .........
 * Step 1: insert / update the record. 

Documentation/Installation.rst

@@ -49,7 +49,7 @@ The following features are only tested / supported on linux hosts:
 * Mime type detection for uploads - command `file`.
 * Split PDF into JPG - command `convert`.
 * Repair PDF - command `pdftocairo`.
-
+* Convert HEIC/HEIF to png - command `heif-info` and `heif-convert`.
 
 .. _`preparation`:
 
@@ -69,7 +69,7 @@ Preparation for Ubuntu::
 
   sudo apt install php-intl
   # for file upload, PDF and 'HTML to PDF' (wkhtmltopdf), PDF split
-  sudo apt install poppler-utils libxrender1 file pdf2svg qpdf ghostscript img2pdf
+  sudo apt install poppler-utils libxrender1 file pdf2svg qpdf ghostscript img2pdf libheif-examples
   sudo apt install inkscape imagemagick            # to render thumbnails
 
 .. _wkhtml:
@@ -424,6 +424,10 @@ Extension Manager: QFQ Configuration
 +-----------------------------------+-------------------------------------------------------+----------------------------------------------------------------------------+
 | cmdImg2pdf                        | img2pdf                                               | PathFilename of img2pdf. Optional variables like LD_LIBRARY_PATH=...       |
 +-----------------------------------+-------------------------------------------------------+----------------------------------------------------------------------------+
+| cmdHeifConvert                    | heif-convert                                          | PathFilename of heif-convert. Optional variables like LD_LIBRARY_PATH=... .|
+|                                   |                                                       | During upload, HEIF/HEIC images will be silently converted to PNG. To      |
+|                                   |                                                       | disable the conversion, leave the field blank.                             |
++-----------------------------------+-------------------------------------------------------+----------------------------------------------------------------------------+
 | sendEMailOptions                  | -o tls=yes                                            | General options. Check: http://caspian.dotconf.net/menu/Software/SendEmail |
 +-----------------------------------+-------------------------------------------------------+----------------------------------------------------------------------------+
 | reportAsFileAutoExport            | no                                                    | Auto export of qfq reports to files. See :ref:`reportAsFile`               |

extension/Classes/Core/Constants.php

@@ -652,6 +652,7 @@ const SYSTEM_CMD_QPDF = 'cmdQpdf';
 const SYSTEM_CMD_GS = 'cmdGs';
 const SYSTEM_CMD_PDFUNITE = 'cmdPdfunite';
 const SYSTEM_CMD_IMG2PDF = 'cmdImg2pdf';
+const SYSTEM_CMD_HEIF_CONVERT = 'cmdHeifConvert';
 
 // Thumbnail
 const SYSTEM_THUMBNAIL_DIR_SECURE_REL_TO_APP = 'thumbnailDirSecure';

extension/Classes/Core/File.php

@@ -8,13 +8,13 @@
 
 namespace IMATHUZH\Qfq\Core;
 
- 
+
+use IMATHUZH\Qfq\Core\Helper\HelperFile;
 use IMATHUZH\Qfq\Core\Helper\Logger;
 use IMATHUZH\Qfq\Core\Helper\Path;
 use IMATHUZH\Qfq\Core\Helper\Support;
 use IMATHUZH\Qfq\Core\Store\Session;
 use IMATHUZH\Qfq\Core\Store\Store;
-use IMATHUZH\Qfq\Core\Helper\HelperFile;
 
 /**
  * Class File
@@ -132,6 +132,7 @@ class File {
         $newArr = reset($_FILES);
         // Merge new upload date to existing status information
         $statusUpload = array_merge($statusUpload, $newArr);
+        HelperFile::fixMimeTypeHeif($statusUpload['tmp_name'], $statusUpload['type']);
 
         if ($statusUpload[FILES_ERROR] !== UPLOAD_ERR_OK) {
             throw new \UserFormException(
@@ -151,6 +152,7 @@ class File {
         // rename uploaded file: ?.cached
         $filenameCached = Support::extendFilename($statusUpload[FILES_TMP_NAME], UPLOAD_CACHED);
 
+        // Remember: PHP['upload_tmp_dir']='' means '/tmp' AND upload process is CHROOT to /tmp/systemd-private-...-apache2.service-../
         error_clear_last();
         if (!move_uploaded_file($newArr[FILES_TMP_NAME], $filenameCached)) {
             $msg = error_get_last();

extension/Classes/Core/Helper/HelperFile.php

@@ -9,8 +9,6 @@
 namespace IMATHUZH\Qfq\Core\Helper;
 
 
-use IMATHUZH\Qfq\Core\Exception\Thrower;
-
 /**
  * Class HelperFile
  * @package qfq
@@ -98,9 +96,32 @@ class HelperFile {
             throw new \UserFormException('Error get mime type of upload.', ERROR_UPLOAD_GET_MIME_TYPE);
         }
 
+        self::fixMimeTypeHeif($pathFilename, $fileMimeType);
+
         return $fileMimeType;
     }
 
+    /**
+     * Checks if the already found MimeType is 'application/octet-stream;'
+     * If yes: Check if the real file is of type HEIC or HEIF by using `heif-info`.
+     * If yes: Set $fileMimeType = 'image/heic; charset=binary';
+     *
+     * @param $pathFilename
+     * @param $fileMimeType
+     */
+    public static function fixMimeTypeHeif($pathFilename, &$fileMimeType) {
+
+        // Workaround for HEIF/HEIC: At least on Ubuntu 18.04 the mime type for HEIC/HEIF is not recognized.
+        // If the false delivered mimetype is detected, check explicitly for HEIC/HEIF
+        if (strpos($fileMimeType, 'application/octet-stream') !== false) {
+            // 'heif-info' is part of the package 'libheif-examples'. If it does not exist: heic/heif is not detected.
+            exec('heif-info ' . $pathFilename, $output, $return_var);
+            if ($return_var == 0) {
+                $fileMimeType = 'image/heic; charset=binary';
+            }
+        }
+    }
+
     /**
      * Returns an array with filestat information to $pathFileName
      * - mimeType
@@ -478,7 +499,7 @@ class HelperFile {
                 ERROR_PDF2JPEG);
         }
 
-        $new = Array();
+        $new = array();
         foreach ($files as $key => $value) {
 
             if ($value == '.' || $value == '..' || $value == QFQ_TEMP_SOURCE) {
@@ -528,8 +549,7 @@ class HelperFile {
      * @param string $fileName
      * @return bool
      */
-    public static function isValidFileName(string $fileName): bool
-    {
+    public static function isValidFileName(string $fileName): bool {
         return preg_match('/^([-\.\w]+)$/', $fileName) > 0;
     }
 
@@ -577,7 +597,7 @@ class HelperFile {
                 ERROR_MESSAGE_TO_DEVELOPER => "Can't write to file '$pathFileName'"]),
                 ERROR_IO_WRITE_FILE);
         }
-        clearstatcache (true, $pathFileName);
+        clearstatcache(true, $pathFileName);
     }
 
     /**
@@ -587,8 +607,7 @@ class HelperFile {
      * @return string
      * @throws \UserFormException
      */
-    public static function file_get_contents(string $pathFileName): string
-    {
+    public static function file_get_contents(string $pathFileName): string {
         $fileContents = file_get_contents($pathFileName);
         if ($fileContents === false) {
             throw new \UserFormException(json_encode([
@@ -606,8 +625,7 @@ class HelperFile {
      * @return array
      * @throws \UserFormException
      */
-    public static function json_decode(string $jsonString, bool $associativeArray = true)
-    {
+    public static function json_decode(string $jsonString, bool $associativeArray = true) {
         $result = json_decode($jsonString, $associativeArray);
         if (is_null($result)) {
             throw new \UserFormException(json_encode([
@@ -624,8 +642,7 @@ class HelperFile {
      * @param $path
      * @throws \UserFormException
      */
-    public static function enforce_writable($path)
-    {
+    public static function enforce_writable($path) {
         if (!is_writeable($path)) {
             throw new \UserFormException(json_encode([
                 ERROR_MESSAGE_TO_USER => 'File/directory not found or not writable.',
@@ -639,8 +656,7 @@ class HelperFile {
      * @param $pathFileName
      * @throws \UserFormException
      */
-    public static function enforce_writable_or_creatable($pathFileName)
-    {
+    public static function enforce_writable_or_creatable($pathFileName) {
         if (file_exists($pathFileName)) {
             self::enforce_writable($pathFileName);
         } else {
@@ -656,8 +672,7 @@ class HelperFile {
      * @return bool
      * @throws \UserFormException
      */
-    public static function isReadableException($pathFileName): bool
-    {
+    public static function isReadableException($pathFileName): bool {
         if (!file_exists($pathFileName)) {
             return false;
         }
@@ -676,8 +691,7 @@ class HelperFile {
      * @return mixed
      * @throws \UserFormException
      */
-    public static function include(string $pathFileName)
-    {
+    public static function include(string $pathFileName) {
         $result = include($pathFileName);
         if (empty($result) || $result === true) {
             throw new \UserFormException(json_encode([

extension/Classes/Core/Save.php

@@ -564,7 +564,6 @@ class Save {
                 $this->store->unsetVar(VAR_FILE_MIME_TYPE, STORE_VAR);
                 $this->store->unsetVar(VAR_FILE_SIZE, STORE_VAR);
             } else {
-
                 $this->store->appendToStore($vars, STORE_VAR);
             }
 
@@ -1155,6 +1154,30 @@ class Save {
             $chmodFile = octdec($formElement[FE_FILE_CHMOD_FILE]);
         }
 
+        if(1){
+            // HEIC conversion?
+            if (strpos($statusUpload['type']??'', 'image/heic') !== false) {
+                $cmd = $this->store->getVar(SYSTEM_CMD_HEIF_CONVERT, STORE_SYSTEM);
+                if (!empty($cmd )) {
+                    exec("$cmd -q 100 $pathFileName $pathFileName.png", $output, $return_var);
+                    if ($return_var == 0) {
+                        HelperFile::unlink($pathFileName);
+                        $pathFileName.='.png';
+                        $statusUpload['type'] = 'image/png';
+                        $statusUpload['name'] .= '.png';
+                    } else {
+                        $msg = error_get_last();
+                        throw new \UserFormException(
+                            json_encode([ERROR_MESSAGE_TO_USER => 'Upload: HEIC conversion failed', ERROR_MESSAGE_TO_DEVELOPER => $msg .'\nFile: '. $pathFileName]),
+                            ERROR_UPLOAD_FILE_TYPE);
+                    }
+                }
+                // Update STORE_VAR
+                $this->store->appendToStore(HelperFile::pathinfo($pathFileName), STORE_VAR);
+                $this->store->setVar(VAR_FILE_DESTINATION, $pathFileName, STORE_VAR);
+            }
+        }
+
         $this->autoOrient($formElement, $pathFileName);
         HelperFile::chmod($pathFileName, $chmodFile);
 

extension/Classes/Core/Store/Config.php

@@ -454,6 +454,7 @@ class Config {
             SYSTEM_CMD_GS => 'gs',
             SYSTEM_CMD_PDFUNITE => 'pdfunite',
             SYSTEM_CMD_IMG2PDF => 'img2pdf',
+            SYSTEM_CMD_HEIF_CONVERT => 'heif-convert',
 
             SYSTEM_THUMBNAIL_DIR_SECURE_REL_TO_APP => Path::APP_TO_SYSTEM_THUMBNAIL_DIR_SECURE_DEFAULT,
             SYSTEM_THUMBNAIL_DIR_PUBLIC_REL_TO_APP => Path::APP_TO_SYSTEM_THUMBNAIL_DIR_PUBLIC_DEFAULT,

extension/Tests/Unit/Core/Store/StoreTest.php

@@ -418,6 +418,7 @@ class StoreTest extends TestCase {
             SYSTEM_CMD_GS => 'gs',
             SYSTEM_CMD_PDFUNITE => 'pdfunite',
             SYSTEM_CMD_IMG2PDF => 'img2pdf',
+            SYSTEM_CMD_HEIF_CONVERT => 'heif-convert',
         ];
 
         $body = json_encode([

extension/ext_conf_template.txt

@@ -40,12 +40,15 @@ cmdQpdf = qpdf
 # cat=config/config; type=string; label=Command 'gs':Default is 'gs'. Will be used to decrypt and to convert PDF to PDF (try to repair) if merging of PDF fails.
 cmdGs = gs
 
-# cat=config/config; type=string; label=Command 'pdfunite':Default is 'pdfunite'. Will be used to merge PDFs.
+# cat=config/config; type=string; label=Command 'pdfunite':Default is 'pdfunite'. Will be used to merge PDF.
 cmdPdfunite = pdfunite
 
-# cat=config/config; type=string; label=Command 'img2pdf':Default is 'img2pdf'. Will be used to convert images to PDFs.
+# cat=config/config; type=string; label=Command 'img2pdf':Default is 'img2pdf'. Will be used to convert images to PDF.
 cmdImg2pdf = img2pdf
 
+# cat=config/config; type=string; label=Command 'heif-convert':Default is 'heif-convert'. Will be used to convert images from HEIC/HEIF to PNG.
+cmdHeifConvert = heif-convert
+
 # cat=config/email; type=string; label=Options for SendEMail:Default is empty. General options. Check: http://caspian.dotconf.net/menu/Software/SendEmail. E.g.: 'sendEMail=-o tls=yes'
 sendEMailOptions =