Autocron: implement a) 'output to a file' (content of the retrived page), b)...

Autocron: implement a) 'output to a file' (content of the retrived page), b) search for a pattern on the output.
Manual.rst: updated to the latest autocron features.

extension/Documentation/Manual.rst

 .. ==================================================
-.. Header hierachy
+.. Header hierarchy
 .. ==
 ..  --
 ..   ^^
@@ -6604,7 +6604,7 @@ The `autocron` service fires periodically jobs like `send a mail` or `open a web
 * Per job: if a job's runs and the receives the next trigger, the running job will be completed first.
 * Per job: if more than one trigger arrives during a run, only one trigger will be processed.
 * If the system misses a run, it will be played as soon as the system is online again.
-* Running and processed jobs can easily be monitored.
+* Running and processed jobs can easily be monitored via *lastRun, lastStatus, nextRun, inProgress*.
 
 Setup
 ^^^^^
@@ -6621,9 +6621,8 @@ Create a T3 page with a QFQ record. Such page should be access restricted and is
 	form={{form:S}}
 
 	10 {
-		 # List of Forms: Do not show this list of forms if there is a form given by SIP.
 		# Table header.
-		sql = SELECT CONCAT('{{pageId:T}}&form=cron') as Pagen, 'id', 'Enable', 'Next run','Frequency','Comment','Last run','Status' FROM (SELECT 1) AS fake WHERE  '{{form:SE}}'=''
+		sql = SELECT CONCAT('p:{{pageId:T}}&form=cron') AS _pagen, 'id', 'Enable', 'Next run','Frequency','Comment','Last run','Status' FROM (SELECT 1) AS fake WHERE '{{form:SE}}'=''
 		head = <table class='table table-hover qfq-table-50'>
 		tail = </table>
 		rbeg = <thead><tr>
@@ -6633,7 +6632,7 @@ Create a T3 page with a QFQ record. Such page should be access restricted and is
 
 		10 {
 			# All Cron Jobs
-		sql = SELECT CONCAT('{{pageId:T}}&form=cron&r=', c.id) AS _Pagee, c.id,
+			sql = SELECT CONCAT('p:{{pageId:T}}&form=cron&r=', c.id) AS _pagee, c.id,
 					  IF(c.status='enable','green','gray') AS _bullet,
 					  IF(c.nextrun=0,"", DATE_FORMAT(c.nextrun, "%d.%m.%y %H:%i:%s")),
 					  c.frequency,
@@ -6651,12 +6650,44 @@ Create a T3 page with a QFQ record. Such page should be access restricted and is
 		}
 	}
 
+Usage
+^^^^^
+
+The OS `cron` service will call the `QFQ autocron` every minute. `QFQ autocron` checks if there is a pending job, by looking
+for jobs with `Next run`<=NOW(). All found jobs will be fired - depending on their type, such jobs will send mail(s) or
+open a `webpage`. A `webpage` will mostly be a local T3 page with at least one QFQ record on it.
+
+After finishing a job, `Next run` will be increased by `Frequency`. If `Next run` still points in the past, it will be
+increased by Frequency until it points to the future.
+
+With `Next run`=0 the auto repeating is switched off.
+
+Type: Mail
+''''''''''
+
+At the moment there is a special sendmail notation - this will change in the future.
+
+* Mail: ::
+
+  {{!SELECT 'john@doe.com' AS sendMailTo, 'Custom subject' AS sendMailSubject, 'jane@doe.com' AS sendMailFrom,
+             123 AS sendMailGrId, 456 AS sendMailXId}}
 
+Autocron will send as many mails as records are selected by the SQL query in field `Mail`. Field `Mail body` provides
+the mail text.
 
 
+Type: Website
+'''''''''''''
 
+The page specified in `URL` will be opened.
 
+Optional the output of the page can be logged to a file (be sure to have write permissions on that file). Also overwrite
+or append can be selected.
 
+To check for a successful DB connection, it's a good practice to report a token on the T3 page / QFQ record like
+'DB Connect: ok'. Such a string can be checked via `Pattern to look for on output`=`/DB Connect: ok/`. The pattern
+needs to be written in PHP PCRE syntax. For a simple search string, just surround them with '/'.
+If the pattern is found on the page, the job get's 'Ok' - else 'Error - ...'.
 
 
 .. _help:

extension/qfq/external/AutoCron.php

@@ -41,9 +41,12 @@ class AutoCron {
 
     private $dbIndexQfq = '';
 
+    private $verbose = '';
 
-    public function __construct($phpUnit = false) {
 
+    public function __construct($verbose = false, $phpUnit = false) {
+
+        $this->verbose = $verbose;
         $this->phpUnit = $phpUnit;
 
         mb_internal_encoding("UTF-8");
@@ -60,7 +63,7 @@ class AutoCron {
     }
 
     /**
-     * Check if there started cronJobs, older than $ageMaxMinutes
+     * Check if there are started cronJobs, older than $ageMaxMinutes
      *
      * @param int $ageMaxMinutes
      *
@@ -68,8 +71,14 @@ class AutoCron {
      * @throws DbException
      */
     private function checkForOldJobs($ageMaxMinutes) {
-        $sql = "SELECT c.id, ' Started at: ', c.inProgress FROM Cron AS c WHERE DATE_ADD(c.inProgress, INTERVAL $ageMaxMinutes MINUTE)<NOW()";
-        $this->dbArray[$this->dbIndexQfq]->sql($sql, ROW_EXPECT_0);
+
+        $sql = "SELECT CONCAT('cron.id=', c.id, ' Started at: ', c.inProgress) FROM Cron AS c WHERE DATE_ADD(c.inProgress, INTERVAL $ageMaxMinutes MINUTE)<NOW() AND c.status='enable' ";
+
+        // If there are too long running jobs: throw an exception
+        $rows = $this->dbArray[$this->dbIndexQfq]->sql($sql, ROW_REGULAR);
+        if (!empty($rows)) {
+            echo implode(PHP_EOL, $rows);
+        }
     }
 
     /**
@@ -113,7 +122,7 @@ class AutoCron {
     }
 
     /**
-     * Calls the Webpage given in $job[AUTOCRON_CONTENT].
+     * Call the web page given in $job[AUTOCRON_CONTENT].
      *
      * @param array $job
      *
@@ -123,7 +132,10 @@ class AutoCron {
 
         $job[AUTOCRON_LAST_STATUS] = 'nothing done';
         $job[AUTOCRON_CONTENT] = trim($job[AUTOCRON_CONTENT]);
+
         if (!empty($job[AUTOCRON_CONTENT])) {
+
+            // If the URL does not start with 'http...': prefix with the site config.
             if (substr($job[AUTOCRON_CONTENT], 0, 4) != 'http') {
                 // We need to prefix
                 $baseUrl = $this->store->getVar(SYSTEM_BASE_URL_PRINT, STORE_SYSTEM);
@@ -133,11 +145,40 @@ class AutoCron {
             // Download page
             $page = file_get_contents($job[AUTOCRON_CONTENT]);
             if ($page === false) {
-                $job[AUTOCRON_LAST_STATUS] = 'Error: failed to fetch ' . $job[AUTOCRON_CONTENT];
-            } else {
+                $job[AUTOCRON_LAST_STATUS] = htmlspecialchars('Error: failed to fetch "' . $job[AUTOCRON_CONTENT] . '"');
+
+                return $job;
+            }
+
+            // If configured, log the download content
+            if (!empty($job[AUTOCRON_OUTPUT_FILE])) {
+
+                if ($job[AUTOCRON_OUTPUT_FILE][0] != '/') {
+                    $job[AUTOCRON_OUTPUT_FILE] .= $this->store->getVar(SYSTEM_SITE_PATH, STORE_SYSTEM);
+                }
+
+                Logger::logMessage($page, $job[AUTOCRON_OUTPUT_FILE], $job[AUTOCRON_OUTPUT_MODE] == 'append' ? FILE_MODE_APPEND : FILE_MODE_WRITE);
+            }
+
+            // Report the first 30 chars
             $job[AUTOCRON_LAST_STATUS] = 'OK: ' . substr($page, 0, 30);
+
+            // Check for pattern?
+            if (!empty($job[AUTOCRON_OUTPUT_PATTERN])) {
+
+                $rc = preg_match($job[AUTOCRON_OUTPUT_PATTERN], $page);
+
+                if ($rc === false) {
+                    $job[AUTOCRON_LAST_STATUS] = 'Error: preg_match() failed - "' . $job[AUTOCRON_OUTPUT_PATTERN] . '"';
+                }
+
+                if ($rc === 0) {
+                    $job[AUTOCRON_LAST_STATUS] = 'Error: pattern not found - "' . $job[AUTOCRON_OUTPUT_PATTERN] . '"';
                 }
             }
+        }
+
+        $job[AUTOCRON_LAST_STATUS] = htmlspecialchars($job[AUTOCRON_LAST_STATUS]);
 
         return $job;
     }

extension/qfq/external/autocron.php

@@ -8,9 +8,16 @@
 
 require_once(__DIR__ . '/AutoCron.php');
 
+global $argv;
+
 try {
+    $verbose = false;
+
+    if (isset($argv[1]) && $argv[1] == '-v') {
+        $verbose = true;
+    }
 
-    $autocron = new \qfq\AutoCron();
+    $autocron = new \qfq\AutoCron($verbose);
     $autocron->process();
 
 } catch (qfq\UserFormException $e) {

extension/qfq/qfq/Constants.php

@@ -1116,6 +1116,9 @@ const FILE_ACTION_DELETE = 'delete';
 const PATH_FILE_CONCAT = 'pathFileConcat';
 const FILE_PRIORITY = 'filePriority';
 
+const FILE_MODE_WRITE = 'w';
+const FILE_MODE_APPEND = 'a';
+
 // DATABASE
 const DB_NUM_ROWS = 'numRows';
 const DB_AFFECTED_ROWS = 'affectedRows';
@@ -1373,6 +1376,9 @@ const AUTOCRON_IN_PROGRESS = 'inProgress';
 const AUTOCRON_STATUS = 'status';
 const AUTOCRON_CONTENT = 'content';
 const AUTOCRON_SQL1 = 'sql1';
+const AUTOCRON_OUTPUT_FILE = 'outputFile';
+const AUTOCRON_OUTPUT_MODE = 'outputMode';
+const AUTOCRON_OUTPUT_PATTERN = 'outputPattern';
 
 const AUTOCRON_UNIT = 'unit';
 const AUTOCRON_COUNT = 'count';

extension/qfq/qfq/database/DatabaseUpdateData.php

@@ -108,6 +108,13 @@ $UPDATE_ARRAY = array(
         "ALTER TABLE `FormElement` CHANGE `label` `label` VARCHAR(511) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT ''",
     ],
 
+    '0.25.16' => [
+        "ALTER TABLE `Cron` ADD `outputFile` VARCHAR(255) NOT NULL AFTER `comment`",
+        "ALTER TABLE `Cron` ADD `outputMode` ENUM('overwrite','append') NOT NULL DEFAULT 'append' AFTER `outputFile`",
+        "ALTER TABLE `Cron` ADD `outputPattern` VARCHAR(255) NOT NULL AFTER `outputMode`",
+    ],
+
+
 );
 
 

extension/qfq/qfq/helper/Logger.php

@@ -21,13 +21,13 @@ class Logger {
      *
      * @throws UserFormException
      */
-    public static function logMessage($msg, $filename) {
+    public static function logMessage($msg, $filename, $mode = FILE_MODE_APPEND) {
 
         if ($filename == '') {
             return;
         }
 
-        if (!$handle = fopen($filename, 'a')) {
+        if (!$handle = fopen($filename, $mode)) {
             throw new UserFormException("Error - cannot open. File: " . $filename . " ( CWD: " . getcwd() . ")", ERROR_IO_OPEN);
         }
 

extension/qfq/sql/formEditor.sql

@@ -416,12 +416,12 @@ CREATE TABLE IF NOT EXISTS `Cron` (
   AUTO_INCREMENT = 0
   DEFAULT CHARSET = utf8;
 
-# Form: Cron
-INSERT INTO Form (id, name, title, tableName, parameter)
+# Form: AutoCron
+INSERT INTO Form (id, name, title, tableName, parameter, dirtyMode)
 VALUES
-  (4, 'cron', 'cron {{SELECT IF("{{r}}" = "0", "(new)", "(Id: {{r}})") }}', 'Cron', '');
+  (4, 'cron', 'cron {{SELECT IF("{{r}}" = "0", "(new)", "(Id: {{r}})") }}', 'Cron', '', 'none');
 
-# FormElements: CopyForm
+# FormElements: AutoCron
 INSERT INTO FormElement (formId, name, label, mode, modeSql, type, encode, checkType, ord, parameter, size, note, dynamicUpdate)
 VALUES
   (4, 'status', 'Enabled', 'show', '', 'checkbox', 'specialchar', 'alnumx', 10, '', '', '', 'no'),
@@ -438,9 +438,14 @@ VALUES
    '60,4', '', 'yes'),
   (4, 'content', '{{SELECT IF("{{type:FR:alnumx}}"="mail","Mail body","URL") }}', 'show', '', 'text', 'none', 'all', 70,
    'extraButtonInfo = Website: URL<br>Mail: Static Body or &#123;{SELECT ...&#125;}', '40,4', '', 'yes'),
-  (4, 'lastRun', 'Last run', 'readonly', '', 'text', 'specialchar', 'alnumx', 80, '', '', '', 'no'),
-  (4, 'lastStatus', 'Laststatus', 'readonly', '', 'text', 'specialchar', 'alnumx', 90, '', '', '', 'no'),
-  (4, 'inProgress', 'Running', 'show', '', 'text', 'specialchar', 'alnumx', 100,
+
+  (4, 'outputFile', 'Log output to file', 'show', '{{SELECT IF("{{type:FR:alnumx}}"="mail","hidden","show") }}', 'text', 'none', 'all', 80, '', '', '', 'yes'),
+  (4, 'outputMode', 'Mode output', 'show', '{{SELECT IF("{{type:FR:alnumx}}"="mail","hidden","show") }}', 'radio', 'specialchar', 'alnumx', 90, 'buttonClass=btn-default', '', '', 'yes'),
+  (4, 'outputPattern', 'Pattern to look for on output', 'show', '{{SELECT IF("{{type:FR:alnumx}}"="mail","hidden","show") }}', 'text', 'none', 'all', 100, '', '', 'If not found return an error. Check <a href="https://secure.php.net/manual/en/pcre.pattern.php">pcre</a> / <a href="https://regexp101.com">regexp101.com</a> ', 'yes'),
+
+  (4, 'lastRun', 'Last run', 'readonly', '', 'text', 'specialchar', 'alnumx', 120, '', '', '', 'no'),
+  (4, 'lastStatus', 'Laststatus', 'readonly', '', 'text', 'specialchar', 'alnumx', 130, '', '', '', 'no'),
+  (4, 'inProgress', 'Running', 'show', '', 'text', 'specialchar', 'alnumx', 140,
    'extraButtonInfo = Starttime of running job. When job is finished, will be set back to 0. A new job will only be started, if this is 0.',
    '', '', 'no');