Async Job TODO

SE/se-lib/Core/AsyncJobs.php

@@ -0,0 +1,400 @@
+<?php
+
+// /opt/local/pl.procesy5/p5build_SE/temp/WPS_Functions/default_db/CRM_PROCES_tree/relations-22
+// /opt/local/pl.procesy5/ -> /Library/New_Server/opt/local/pl.procesy5
+// /opt/local/pl.procesy5/async_jobs - TODO: APP_PATH_ASYNC_JOB from .htaccess or config
+
+// Config example:
+// cat SE/config/.cnf-biuro.biall-net.pl.ini.php:
+// APP_PATH_ASYNC_JOB="/opt/local/pl.procesy5/async_jobs"
+// register new job:
+// - $TODAY = date("Y-m-d");
+// - $JOB_ID = generateJobID if not already started?
+// - make folder APP_PATH_ASYNC_JOB / $TODAY / job-{$JOB_ID}
+// - make base files:
+//   - log files: `out.log`, `error.log`
+//   - `output`: if process creates file. @param --out_file=out
+//   - `output_type`: output mime type or namespace
+//   - `progress`: if process implement progress. @param --progress_file=progress
+
+class Core_AsyncJobs {
+	static $VERSION = 1; // `CRM_CONFIG`.`CONF_KEY` = 'Core_AsyncJobs__version'
+	static $CRM_CONFIG_VERSION_KEY = 'Core_AsyncJobs__version';
+
+	static function getSimpleList() {
+		$fullList = self::getFullList();
+		return array_map(function ($jobInfo) {
+			return [
+				'name' => $jobInfo['name'],
+				'pid' => $jobInfo['pid'],
+				'pm_id' => $jobInfo['pm_id'],
+				'pm2_env.status' => $jobInfo['pm2_env']['status'],
+				'monit.memory' => $jobInfo['monit']['memory'],
+				'monit.cpu' => $jobInfo['monit']['cpu'],
+				//   "pm_out_log_path": "/Library/WebServer/.pm2/logs/test-job-1-out.log",
+				//   "pm_err_log_path": "/Library/WebServer/.pm2/logs/test-job-1-error.log",
+				//   "pm_pid_path": "/Library/WebServer/.pm2/pids/test-job-0.pid",
+				'pm2_env.pm_out_log_path' => $jobInfo['pm2_env']['pm_out_log_path'],
+				'pm2_env.pm_err_log_path' => $jobInfo['pm2_env']['pm_err_log_path'],
+				'pm2_env.pm_pid_path' => $jobInfo['pm2_env']['pm_pid_path'],
+			];
+		}, $fullList);
+	}
+
+	static function getFullList() {
+		$jobListJson = V::shell_exec("pm2 jlist 2>&1");
+		if (empty($jobListJson)) throw new Exception("Reading async job list failed");
+		$parsedJobList = @json_decode($jobListJson, $assoc = true);
+		if (null == $parsedJobList && 0 !== json_last_error()) throw new Exception("Parsing async job list failed: " . json_last_error());
+		return $parsedJobList;
+	}
+
+	static function getJobLogs($jobNameOrID, $lastLines = 10) {
+		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in getJobLogs");
+		$lastLines = ((int)$lastLines > 0) ? (int)$lastLines : 10;
+		// $cmd = "pm2 logs {$jobNameOrID} --lines {$lastLines} --nostream | tail -n {$lastLines}";
+		$cmd = "pm2 logs {$jobNameOrID} --lines {$lastLines} --nostream";
+		return V::shell_exec($cmd);
+		// [TAILING] Tailing last 3 lines for [0] process (change the value with --lines option)
+		// /Library/WebServer/.pm2/logs/test-job-1-out.log last 3 lines:
+		// /Library/WebServer/
+	}
+
+	static function stopJob($jobNameOrID) {
+		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in stopJob");
+		$cmd = "pm2 stop {$jobNameOrID}";
+		return V::shell_exec($cmd);
+	}
+
+	static function startJob($jobNameOrID) {
+		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in startJob");
+		$cmd = "pm2 start {$jobNameOrID}";
+		return V::shell_exec($cmd);
+	}
+
+	static function startNewJob($jobName) {
+		// for Ant: 
+		// index.php?_route=UrlAction_Ant&_task=ant&path=default_db.in7_dziennik_koresp/etykieta&typeName=default_db:IN7_DZIENNIK_KORESP&primaryKey=66263&primaryKeyField=ID
+		// index.php?_route=UrlAction_Ant
+		// 	& _task=ant
+		// 	& path=default_db.in7_dziennik_koresp/test-bash
+		// 	& typeName=default_db:IN7_DZIENNIK_KORESP
+		// 	& primaryKey=66263
+		// 	& primaryKeyField=ID
+		// index.php?_route=UrlAction_Ant
+		// 	& _task=ant
+		// 	& path=default_db.in7_dziennik_koresp/test-bash
+		// 	& template=test-loop
+		// 	& typeName=default_db:IN7_DZIENNIK_KORESP
+		// 	& primaryKey=66263
+		// 	& primaryKeyField=ID
+		// ant=default_db:IN7_DZIENNIK_KORESP/test-bash & task=test-loop & ns=default_db:IN7_DZIENNIK_KORESP & pk=66263
+		if (empty($jobName)) throw new Exception("Missing job name");
+		// $jobName - check if already started? pm2 will return failed
+		$outLogPath = "p5-async-jobs/jobX/logs/out.log";
+		$errorLogPath = "p5-async-jobs/jobX/logs/error.log";
+		$jobExecPath = ""; // TODO: path to exec
+		$args = ""; // args for script
+		$cmd = implode(" ", [
+			"pm2 start '{$jobExecPath}'",
+			"--name '{$jobName}'",
+			"--no-autorestart",
+			"--output '{$outLogPath}'",
+			"--error '{$errorLogPath}'",
+			"--time", // prefix time to log entry
+			"-- {$args}",
+		]);
+		return V::shell_exec($cmd);
+	}
+
+	static function deleteStopped() {
+		// TODO: script to remove stopped in loop with delay
+		// pm2 start app.js --restart-delay=3000
+		// $ pm2 list | grep '^│' |  awk -F'│' '{ gsub(/ /, "", $2); gsub(/ /, "", $10); if ("stopped" == $10) { print $2" # STOPPED" } else { print $10 } }' |  grep '# STOPPED' | xargs -n1 pm2 delete
+		$testCmd = implode(" | ", [
+			"pm2 list",
+			"grep '^│'",
+			"awk -F'│' '{ gsub(/ /, \"\", \$2); gsub(/ /, \"\", \$10); if (\"stopped\" == \$10) { print \$2\" # STOPPED\" } else { print \$10 } }'",
+			"grep '# STOPPED'",
+			"awk '{print \$1}'",
+		]);
+		V::exec($testCmd . " 2>&1", $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+
+		$cmd = implode(" | ", [
+			"pm2 list",
+			"grep '^│'",
+			"awk -F'│' '{ gsub(/ /, \"\", \$2); gsub(/ /, \"\", \$10); if (\"stopped\" == \$10) { print \$2\" # STOPPED\" } else { print \$10 } }'",
+			"grep '# STOPPED'",
+			"awk '{print \$1}'",
+			"xargs -n1 pm2 delete",
+		]);
+	}
+
+	static function getNodePath() { return "/usr/local/bin/node"; }
+	// npm_path="/usr/local/bin/npm"
+	static function getPm2Path() { return "/usr/local/bin/pm2"; }
+	static function getPm2WwwUserPath() { return "/Library/WebServer/.pm2/"; }
+	static function isInstalled() {
+		$confAsyncPath = Config::get('APP_PATH_ASYNC_JOB');
+		if (!$confAsyncPath) throw new Exception("Missing Config APP_PATH_ASYNC_JOB");
+
+		if (!file_exists($confAsyncPath)) {
+			mkdir($confAsyncPath, $mode = 0777, $recursive = TRUE);
+		}
+		if (!file_exists($confAsyncPath)) throw new Exception("Folder not exists APP_PATH_ASYNC_JOB");
+
+		// V::exec("/usr/local/bin/pm2 --version 2>&1", $out, $ret);
+		V::exec("/usr/local/bin/pm2 ping 2>&1", $out, $ret); // expected "{ msg: 'pong' }"
+		// echo UI::h('pre', [], "ret({$ret}):\n" . implode("\n", $out));
+		if ($ret === 0) {
+			// [PM2] Spawning PM2 daemon with pm2_home=/Library/WebServer/.pm2
+			// [PM2] PM2 Successfully daemonized
+			// { msg: 'pong' }
+		}
+		if ($ret !== 0) {
+			if (!file_exists(self::getNodePath())) throw new Exception("pm2 not installed");
+			// if [ ! -f "$node_path" ]; then
+			// 	echo "$node_path not exists"
+			// 	wget https://nodejs.org/dist/v12.15.0/node-v12.15.0.pkg
+			// 	sudo installer -verbose -pkg node-v12.15.0.pkg -target /
+			// fi
+			// # node -v # expected v12.15.0
+			if (!file_exists(self::getPm2Path())) throw new Exception("pm2 not installed");
+			// npm install -g pm2
+			// sudo npm install -g pm2
+			// pm2 -version # expected 4.2.3
+			if (!file_exists(self::getPm2WwwUserPath())) throw new Exception("pm2 user folder not exists");
+			// FIX for Mac OS:
+			// $ sudo mkdir /Library/WebServer/.pm2/
+			// $ sudo chown _www /Library/WebServer/.pm2/
+			throw new Exception("Error pm2"); // unknown error
+		}
+
+		if (!self::checkAsyncJobDatabase()) throw new Exception("Error database schema for AsyncJobs");
+
+		return true;
+	}
+
+	static function checkAsyncJobDatabase() {
+		// `CRM_CONFIG`.`CONF_KEY` = 'Core_AsyncJobs__version'
+		$dbVersion = (int)DB::getPDO()->fetchValue(" select CONV_VAL from CRM_CONFIG where CONF_KEY = :key ", [ ':key' => self::$CRM_CONFIG_VERSION_KEY ]);
+		if ($dbVersion < 1) self::upgradeAsyncJobDatabaseToVersion1();
+
+		// $dbVersion = (int)DB::getPDO()->fetchValue(" select CONV_VAL from CRM_CONFIG where CONF_KEY = :key ", [ ':key' => 'Core_AsyncJobs__version' ]);
+		// if ($dbVersion < 2) self::upgradeAsyncJobDatabaseToVersion2();
+
+		$dbVersion = (int)DB::getPDO()->fetchValue(" select CONV_VAL from CRM_CONFIG where CONF_KEY = :key ", [ ':key' => self::$CRM_CONFIG_VERSION_KEY ]);
+
+		return ($dbVersion < self::$VERSION) ? false : true;
+	}
+	static function upgradeAsyncJobDatabaseToVersion1() {
+
+		// - insertOrUpdate new row in `CRM_ASYNC_FUNCTIONS` ( $TODAY, $version, $user, $jobName )
+		// - $JOB_ID = fetchValue select ID from `CRM_ASYNC_FUNCTIONS` where JOB_NAME = $jobName
+		//   - `CRM_ASYNC_FUNCTIONS`.`A_STATUS` default 'WAITING' - not started
+		//   - `CRM_ASYNC_FUNCTIONS`.`A_STATUS`: 'NORMAL' - started
+		//   - `CRM_ASYNC_FUNCTIONS`.`A_STATUS`: 'OFF_HARD' - not running
+		//   - `CRM_ASYNC_FUNCTIONS`.`A_STATUS`: 'DELETED' - removed
+		$sql = "
+			CREATE TABLE IF NOT EXISTS `CRM_ASYNC_FUNCTIONS` ( -- list of async function definitions / config
+				`ID` int(11) NOT NULL AUTO_INCREMENT,
+				`ID_ZASOB` int(11) NOT NULL DEFAULT 0, -- TODO - register function in CRM_LISTA_ZASOBOW URL_ACTION to set perms
+				`JOB_NAME` varchar(200) NOT NULL,
+				`VERSION` int(11) NOT NULL DEFAULT 0,
+				`LOCK_TYPE` enum('ROW', 'TABLE', 'SYSTEM', 'NO_LOCK') DEFAULT 'ROW',
+				-- ROW - only one active job per row, eg. create pdf, close FV, @require primaryKey in JOB.LOCK_VALUE
+				-- TABLE - ony one active job per table, eg. update columns, make report, sync
+				-- SYSTEM - only one active job
+				-- TODO USER_... - lock per user
+				-- NO_LOCK - allow multiple jobs
+
+		--		`USER` varchar(20) NOT NULL,
+		--		`DATE` date NOT NULL,
+		--		`A_STATUS` enum('WAITING','NORMAL','WARNING','OFF_SOFT','OFF_HARD','DELETED') DEFAULT 'WAITING',
+
+				`A_RECORD_CREATE_DATE` datetime NOT NULL,
+				`A_RECORD_CREATE_AUTHOR` varchar(20) NOT NULL,
+				`A_RECORD_UPDATE_DATE` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+				`A_RECORD_UPDATE_AUTHOR` varchar(20) NOT NULL,
+				PRIMARY KEY (`ID`),
+				UNIQUE KEY `JOB_NAME` (`JOB_NAME`),
+				KEY `DATE` (`DATE`),
+				KEY `A_STATUS` (`A_STATUS`),
+				KEY `A_RECORD_UPDATE_DATE` (`A_RECORD_UPDATE_DATE`)
+			) ENGINE=MyISAM DEFAULT CHARSET=latin2;
+		";
+		$sql = "
+			CREATE TABLE IF NOT EXISTS `CRM_ASYNC_JOB_LOG` (
+				`ID` int(11) NOT NULL AUTO_INCREMENT,
+				`ID_FUNCTION` int(11) NOT NULL,
+				`ID_SOURCE_JOB` int(11) NOT NULL DEFAULT 0, -- id job
+				`P_ID` int(11) NOT NULL DEFAULT 0, -- parent job id
+				`DATE` date NOT NULL,
+				`VERSION` int(11) NOT NULL DEFAULT 0,
+				`JOB_NAME` varchar(200) NOT NULL, -- copy from CRM_ASYNC_FUNCTIONS
+				`LOCK_VALUE` varchar(200) NOT NULL DEFAULT '', -- second part for unique JOB_NAME eg. primaryKey
+				`USER` varchar(20) NOT NULL, -- user who start this function
+				`A_STATUS` enum('WAITING','NORMAL','WARNING','OFF_SOFT','OFF_HARD','DELETED') DEFAULT 'WAITING',
+
+				`A_RECORD_CREATE_DATE` datetime NOT NULL,
+				`A_RECORD_CREATE_AUTHOR` varchar(20) NOT NULL,
+				`A_RECORD_UPDATE_DATE` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+				`A_RECORD_UPDATE_AUTHOR` varchar(20) NOT NULL,
+				PRIMARY KEY (`ID`),
+				UNIQUE KEY `JOB_LOCK` (`JOB_NAME`, `LOCK_VALUE`, `USER`), -- add USER to store user click and respond that job is running by other user
+				KEY `DATE` (`DATE`),
+				KEY `A_STATUS` (`A_STATUS`),
+				KEY `A_RECORD_UPDATE_DATE` (`A_RECORD_UPDATE_DATE`)
+			) ENGINE=MyISAM DEFAULT CHARSET=latin2;
+		";
+
+		// @usage:
+		// `CRM_ASYNC_FUNCTIONS`.`JOB_NAME` = '
+		//     route = ant
+		// --  & path = default_db.in7_dziennik_koresp/test-bash
+		//     & template = test-loop
+		//     & typeName = default_db:IN7_DZIENNIK_KORESP
+		//     & primaryKey = 66263
+		//     & primaryKeyField = ID
+		// ->  `CRM_ASYNC_FUNCTIONS`.`JOB_NAME` =
+		//     namespace = default_db/IN7_DZIENNIK_KORESP
+		//     ant = test-bash
+		//     template = test-loop
+		//     primaryKey = 66263
+		// ->  `CRM_ASYNC_FUNCTIONS`.`JOB_NAME` Format: "{$type}|..."
+		// ->  `CRM_ASYNC_FUNCTIONS`.`JOB_NAME` Format: "Ant|{$namespace}"
+		// ->  `CRM_ASYNC_JOB_LOG`.`LOCK_VALUE` Format: "{$primaryKey}|{$func_name}|..."
+		// ->  `CRM_ASYNC_JOB_LOG`.`LOCK_VALUE` Format: "{$primaryKey}|{$func_name}|{$template}"
+		// -> Example: JOB_NAME = 'Ant|default_db/IN7_DZIENNIK_KORESP', LOCK_VALUE = '66263|test-bash|test-loop'
+
+		// Add fields:
+		// - function config at execution time:
+		//   - eg LOCK by user | feature | namespace | no_lock -- should contain in name
+		// - process result - last result - NO - result from file, separate request needed
+
+		// IDEA: 2 tables:
+		// - `Config` - for function config (create row if not exists)
+		// - `Log` - for log where JOB_ID is created or used from Config table
+
+		// then createNewJob:
+		// 1. fetch job config from CRM_ASYNC_FUNCTIONS_CONFIG
+		// 1.1 IF 404 then create row, read config from function (how?)
+		// 1.2 IF exists then check status and lock config
+		// 1.2.1 IF online and lock then return error
+		// 1.2.2 IF !online and !lock then create new row in Log table and return correct JOB_ID
+
+		self::_upgdateAsyncJobDatabaseVersion($version = 1);
+	}
+	static function upgradeAsyncJobDatabaseToVersion2() {
+		// ...
+
+		self::_upgdateAsyncJobDatabaseVersion($version = 2);
+	}
+	static function _upgdateAsyncJobDatabaseVersion($version) {
+		DB::getPDO()->insertOrUpdate('CRM_CONFIG', [
+			'CONF_KEY' => self::$CRM_CONFIG_VERSION_KEY,
+			'@insert' => [
+				'CONF_VAL' => $version,
+			],
+			'@update' => [
+				'CONF_VAL' => $version,
+			]
+		]);
+	}
+
+	// pm2 logs -h
+	//
+	//   Usage: logs [options] [id|name]
+	//   stream logs file. Default stream all logs
+	//   Options:
+	//     --json                json log output
+	//     --format              formated log output
+	//     --raw                 raw output
+	//     --err                 only shows error output
+	//     --out                 only shows standard output
+	//     --lines <n>           output the last N lines, instead of the last 15 by default
+	//     --timestamp [format]  add timestamps (default format YYYY-MM-DD-HH:mm:ss)
+	//     --nostream            print logs without lauching the log stream
+	//     --highlight [value]   highlights the given value
+	//     -h, --help            output usage information
+
+
+	// [0] => Array:
+	//     [pid] => 71728
+	//     [name] => test-job-1
+	//     [pm2_env] => Array:
+	//         [exit_code] => 0
+	//         [versioning] => 
+	//         [version] => N/A
+	//         [unstable_restarts] => 0
+	//         [restart_time] => 53
+	//         [pm_id] => 0
+	//         [created_at] => 1581938950672
+	//         [axm_dynamic] => Array:
+	//         [axm_options] => Array:
+	//         [axm_monitor] => Array:
+	//         [axm_actions] => Array:
+	//         [pm_uptime] => 1581939524854
+	//         [status] => online
+	//         [unique_id] => 80fefcff-8605-44cc-9829-bbcba4de4e7c
+	//         [PM2_HOME] => /Library/WebServer/.pm2
+	//         [PATH] => /usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/local/bin:/opt/local/lib/mysql55/bin:/Applications/Server.app/Contents/ServerRoot/usr/bin:/Applications/Server.app/Contents/ServerRoot/usr/sbin:/Users/pl/programy/bin
+	//         [PWD] => /Users/plabudda/rsync.se.master/SE
+	//         [XPC_FLAGS] => 0x80
+	//         [XPC_SERVICE_NAME] => 0
+	//         [HOME] => /Library/WebServer
+	//         [SHLVL] => 1
+	//         [SERVER_INSTALL_PATH_PREFIX] => /Applications/Server.app/Contents/ServerRoot
+	//         [XPC_SERVICES_UNAVAILABLE] => 1
+	//         [MODULE_INSTALL_PATH_PREFIX] => 
+	//         [_] => /usr/local/bin/pm2
+	//         [__CF_USER_TEXT_ENCODING] => 0x46:0:0
+	//         [PM2_USAGE] => CLI
+	//         [NODE_APP_INSTANCE] => 0
+	//         [vizion_running] => 
+	//         [km_link] => 
+	//         [pm_pid_path] => /Library/WebServer/.pm2/pids/test-job-0.pid
+	//         [pm_err_log_path] => /Library/WebServer/.pm2/logs/test-job-1-error.log
+	//         [pm_out_log_path] => /Library/WebServer/.pm2/logs/test-job-1-out.log
+	//         [instances] => 1
+	//         [exec_mode] => fork_mode
+	//         [exec_interpreter] => php
+	//         [pm_cwd] => /Users/plabudda/rsync.se.master/SE
+	//         [pm_exec_path] => /Users/plabudda/rsync.se.master/sbin/test-sleep-loop.php
+	//         [node_args] => Array:
+	//         [name] => test-job-1
+	//         [namespace] => default
+	//         [env] => Array:
+	//             [unique_id] => 80fefcff-8605-44cc-9829-bbcba4de4e7c
+	//             [test-job-1] => Array:
+	//             [PM2_HOME] => /Library/WebServer/.pm2
+	//             [PATH] => /usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/local/bin:/opt/local/lib/mysql55/bin:/Applications/Server.app/Contents/ServerRoot/usr/bin:/Applications/Server.app/Contents/ServerRoot/usr/sbin:/Users/pl/programy/bin
+	//             [PWD] => /Users/plabudda/rsync.se.master/SE
+	//             [XPC_FLAGS] => 0x80
+	//             [XPC_SERVICE_NAME] => 0
+	//             [HOME] => /Library/WebServer
+	//             [SHLVL] => 1
+	//             [SERVER_INSTALL_PATH_PREFIX] => /Applications/Server.app/Contents/ServerRoot
+	//             [XPC_SERVICES_UNAVAILABLE] => 1
+	//             [MODULE_INSTALL_PATH_PREFIX] => 
+	//             [_] => /usr/local/bin/pm2
+	//             [__CF_USER_TEXT_ENCODING] => 0x46:0:0
+	//             [PM2_USAGE] => CLI
+	//         [merge_logs] => 1
+	//         [vizion] => 1
+	//         [autorestart] => 1
+	//         [watch] => 
+	//         [instance_var] => NODE_APP_INSTANCE
+	//         [pmx] => 1
+	//         [automation] => 1
+	//         [treekill] => 1
+	//         [username] => _www
+	//         [windowsHide] => 1
+	//         [kill_retry_time] => 100
+	//     [pm_id] => 0
+	//     [monit] => Array:
+	//         [memory] => 15720448
+	//         [cpu] => 0
+
+}

SE/se-lib/Route/AsyncJobs.php

@@ -0,0 +1,268 @@
+<?php
+
+Lib::loadClass('RouteBase');
+Lib::loadClass('Core_AsyncJobs');
+
+class Route_AsyncJobs extends RouteBase {
+
+	function isInstalled() { return Core_AsyncJobs::isInstalled(); }
+
+	function defaultAction() { UI::layout([ $this, 'defaultView' ]); }
+	function defaultView() {
+		echo UI::h('h1', [], "Async jobs (wps server)");
+		if (!User::isAdmin()) throw new Exception("Access Denied");
+
+		DBG::nicePrint(Config::getConfFile(), "Config::getConfFile()");
+
+		echo UI::h('div', [ 'style' => "padding:12px 0; border-bottom:1px solid #ddd" ], [
+			UI::hButtonPost("Check", [
+				'class' => "btn btn-default",
+				'title' => "check if async process manager is working",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "pm2Check",
+				]
+			]),
+			" ",
+			UI::h('a', [
+				'href' => $this->getLink('list'),
+				'class' => "btn btn-default",
+			], "Lista"),
+			" ",
+
+			UI::hButtonPost("pm2 list", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "pm2List",
+				]
+			]),
+			" ",
+			UI::hButtonPost("pm2 kill", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "pm2Kill",
+				]
+			]),
+			" ",
+			UI::hButtonPost("pm2 run1", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "mp2Start1",
+				]
+			]),
+			" ",
+			UI::hButtonPost("pm2 run2", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "mp2Start2",
+				]
+			]),
+			" ",
+			UI::hButtonPost("pm2 run3", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "mp2Start3",
+				]
+			]),
+			" ",
+			UI::hButtonPost("pm2 delete stopped", [
+				'class' => "btn btn-warning",
+				'data' => [
+					'_route' => 'AsyncJobs',
+					'_postTask' => "mp2DeleteStopped",
+				]
+			]),
+		]);
+
+		$postTask = V::get('_postTask', '', $_POST);
+		switch ($postTask) {
+			case "pm2Check": UI::tryCatchView([ $this, 'pm2CheckPostTask' ]); break;
+			case "pm2List": UI::tryCatchView([ $this, 'pm2ListPostTask' ]); break;
+			case "pm2Kill": UI::tryCatchView([ $this, 'pm2KillPostTask' ]); break;
+			case "mp2Start1": UI::tryCatchView([ $this, 'mp2Start1PostTask' ]); break;
+			case "mp2Start2": UI::tryCatchView([ $this, 'mp2Start2PostTask' ]); break;
+			case "mp2Start3": UI::tryCatchView([ $this, 'mp2Start3PostTask' ]); break;
+			case "mp2DeleteStopped": UI::tryCatchView([ $this, 'mp2DeleteStoppedPostTask' ]); break;
+		}
+
+	}
+
+	function pm2CheckPostTask() {
+		$cmd = "pm2 ping";
+		V::exec($cmd, $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+		// expected: end($out) === "{ msg: 'pong' }"
+	}
+	function listAction() { UI::layout([ $this, 'listView' ]); }
+	function listView() {
+		echo UI::h('ol', [ 'class' => "breadcrumb", 'style' => "font-size:14px" ], [
+			UI::h('li', [], [
+				UI::h('i', [ 'class' => "glyphicon glyphicon-home" ]),
+				" ",
+				UI::h('a', [ 'href' => $this->getLink() ], "Async jobs"),
+			]),
+			UI::h('li', [ 'class' => "active" ], [
+				"Lista"
+			]),
+			UI::h('a', [
+				'href' => $this->getLink('list'),
+				'class' => "pull-right btn btn-link",
+				'style' => "padding:0; line-height:1em",
+			], "odśwież"),
+		]);
+
+		$postTask = V::get('_postTask', '', $_POST);
+		switch ($postTask) {
+			case "stopJob": UI::tryCatchView([ $this, 'stopJobPostTask' ]); break;
+			case "startJob": UI::tryCatchView([ $this, 'startJobPostTask' ]); break;
+		}
+
+		echo UI::hTable([
+			'rows' => array_map([ $this, 'viewTableRowAsyncJob' ], Core_AsyncJobs::getSimpleList()),
+		]);
+	}
+	function stopJobPostTask() {
+		$idJob = V::get('idJob', 0, $_POST, 'int'); // $jobInfo['pm_id'],
+		$name = V::get('name', '', $_POST); // $jobInfo['name'],
+		if ($idJob < 0) throw new Exception("Wrong param idJob");
+		if ($idJob === 0 && $_POST['idJob'] !== '0') throw new Exception("Wrong param idJob 0");
+
+		Core_AsyncJobs::stopJob($idJob);
+	}
+	function startJobPostTask() {
+		$idJob = V::get('idJob', 0, $_POST, 'int'); // $jobInfo['pm_id'],
+		$name = V::get('name', '', $_POST); // $jobInfo['name'],
+		if ($idJob < 0) throw new Exception("Wrong param idJob");
+		if ($idJob === 0 && $_POST['idJob'] !== '0') throw new Exception("Wrong param idJob 0");
+
+		Core_AsyncJobs::startJob($idJob);
+	}
+	function pm2ListPostTask() {
+		$cmd = "pm2 list";
+		V::exec($cmd, $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+		$cmd = "pm2 jlist";
+		V::exec($cmd, $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+
+		echo '<hr style="margin:24px auto">';
+		$parsedJobList = Core_AsyncJobs::getFullList();
+		DBG::nicePrint($parsedJobList, "\$parsedJobList");
+		echo '<hr style="margin:24px auto">';
+
+		UI::table([
+			'rows' => array_map([ $this, 'viewTableRowAsyncJob' ], Core_AsyncJobs::getSimpleList()),
+		]);
+	}
+	function viewTableRowAsyncJob($jobInfo) {
+		return [
+			'name' => $jobInfo['name'],
+			'pid' => $jobInfo['pid'],
+			'pm_id' => $jobInfo['pm_id'],
+			'status' => $jobInfo['pm2_env.status'],
+			'memory' => $jobInfo['monit.memory'],
+			'cpu' => $jobInfo['monit.cpu'],
+			'#' => UI::h(null, [], [
+				UI::h('a', [
+					'href' => $this->getLink('jobLog', [
+						'idJob' => $jobInfo['pm_id'],
+						'name' => $jobInfo['name'],
+					]),
+					'class' => "btn btn-xs btn-warning",
+				], "logi"),
+				" ",
+				UI::hButtonPost("stop", [
+					'class' => "btn btn-xs btn-default",
+					'data' => [
+						'_route' => 'AsyncJobs',
+						'_postTask' => "stopJob",
+						'idJob' => $jobInfo['pm_id'],
+						'name' => $jobInfo['name'],
+					]
+				]),
+				" ",
+				UI::hButtonPost("start", [
+					'class' => "btn btn-xs btn-default",
+					'data' => [
+						'_route' => 'AsyncJobs',
+						'_postTask' => "startJob",
+						'idJob' => $jobInfo['pm_id'],
+						'name' => $jobInfo['name'],
+					]
+				]),
+			]),
+		];
+	}
+
+	function jobLogAction() { UI::layout([ $this, 'jobLogView' ]); }
+	function jobLogView() {
+		$idJob = V::get('idJob', 0, $_GET, 'int'); // $jobInfo['pm_id'],
+		$name = V::get('name', '', $_GET); // $jobInfo['name'],
+
+		echo UI::h('ol', [ 'class' => "breadcrumb", 'style' => "font-size:14px" ], [
+			UI::h('li', [], [
+				UI::h('i', [ 'class' => "glyphicon glyphicon-home" ]),
+				" ",
+				UI::h('a', [ 'href' => $this->getLink() ], "Async jobs"),
+			]),
+			UI::h('li', [], [
+				UI::h('a', [ 'href' => $this->getLink('list') ], "Lista"),
+			]),
+			UI::h('li', [ 'class' => "active" ], [
+				"Log dla '{$idJob}'",
+			]),
+		]);
+
+		if ($idJob < 0) throw new Exception("Wrong param idJob");
+		if ($idJob === 0 && $_GET['idJob'] !== '0') throw new Exception("Wrong param idJob 0");
+
+		$out = Core_AsyncJobs::getJobLogs($idJob, 10);
+		echo UI::h('div', [ 'style' => "background-color:#ddd; padding:4px 12px" ], "Logi dla zadania '{$idJob}'");
+		echo UI::h('pre', [ 'style' => "border:1px solid #ddd; border-radius:0; padding:12px" ], $out);
+	}
+
+
+	function pm2KillPostTask() {
+		$cmd = "pm2 kill";
+		V::exec($cmd, $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+	}
+	function mp2Start1PostTask() {
+		$cmd = "pm2 start " . APP_PATH_ROOT . "/../sbin/test-sleep-loop.php --name='test-job-1'";
+		V::exec($cmd . " 2>&1", $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+	}
+	function mp2Start2PostTask() {
+		$cmd = "pm2 start " . APP_PATH_ROOT . "/../sbin/test-sleep-loop.php --name='test-job-2'";
+		V::exec($cmd . " 2>&1", $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+	}
+	function mp2Start3PostTask() {
+		$jobName = implode("&", [
+			"ant=default_db.in7_dziennik_koresp/test-bash",
+			"task=test-loop",
+			"ns=default_db:IN7_DZIENNIK_KORESP",
+			"pk=66263",
+		]);
+		$jobNamespace = "p5";
+		$cmd = implode(" ", [
+			"pm2 start",
+			APP_PATH_ROOT . "/../sbin/test-sleep-loop.php",
+			"--name='{$jobName}'",
+			"--namespace='{$jobNamespace}'",
+		]);
+		V::exec($cmd . " 2>&1", $out, $ret);
+		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";
+	}
+
+	function mp2DeleteStoppedPostTask() {
+		Core_AsyncJobs::deleteStopped();
+		UI::alert("DONE");
+	}
+
+}

SE/se-lib/Route/Status.php

@@ -7,34 +7,30 @@ Lib::loadClass('SchemaFactory');
 
 class Route_Status extends RouteBase {
 
-	public function defaultAction() {
-		UI::gora();
-		UI::startTag('div', ['class' => "container"]);
-		echo UI::h('h1', [], [
-			UI::h('a', ['href'=>"index.php"], "SE"),
-			" » ",
-			" Status systemu procesy5"
+	public function defaultAction() { UI::layout([ $this, 'defaultView' ]); }
+	public function defaultView() {
+		echo UI::h('ol', [ 'class' => "breadcrumb" ], [
+			UI::h('li', [], [
+				UI::h('a', [ 'href' => "index.php" ], "SE"),
+			]),
+			UI::h('li', [ 'class' => "active" ], "Status"),
 		]);
-		try {
-			DB::getPDO();
+		echo UI::h('h1', [], "Status systemu procesy5");
 
-			$this->runPostTask(); // _postTask = $this->"{$_POST['_postTask']}PostTask"()
+		DB::getPDO();
 
-			$this->viewStatusDatabase();
+		$this->runPostTask(); // _postTask = $this->"{$_POST['_postTask']}PostTask"()
 
-			if (in_array(User::get('ADM_ADMIN_LEVEL'), ['0', '1'])) {
-				$this->viewStatusUsers();
-			}
+		$this->viewStatusMain();
 
-			// UI::table([
-			//	 'caption' => 'Baza danych',
-			//	 'rows' => DB::getPDO()->fetchAll(" SHOW VARIABLES ")
-			// ]);
-		} catch (Exception $e) {
-			UI::alert('danger', $e->getMessage());
+		if (in_array(User::get('ADM_ADMIN_LEVEL'), ['0', '1'])) {
+			$this->viewStatusUsers();
 		}
-		UI::endTag('div');// .container
-		UI::dol();
+
+		// UI::table([
+		//	 'caption' => 'Baza danych',
+		//	 'rows' => DB::getPDO()->fetchAll(" SHOW VARIABLES ")
+		// ]);
 	}
 	function runPostTask() {
 		if ($postTask = V::get('_postTask', '', $_REQUEST)) {
@@ -52,7 +48,7 @@ class Route_Status extends RouteBase {
 		}
 	}
 
-	public function viewStatusDatabase() {
+	public function viewStatusMain() {
 		$dbEvents = DB::getPDO()->fetchFirst(" SHOW VARIABLES WHERE VARIABLE_NAME = 'event_scheduler' ");
 		// DBG::nicePrint($dbEvents, '$dbEvents');
 		//	 [Variable_name] => event_scheduler
@@ -75,6 +71,14 @@ class Route_Status extends RouteBase {
 		Lib::loadClass('RefConfig');
 		$totalToUpdateRef = RefConfig::getToUpdateTotal();
 
+		$isAsyncJobBackendInstalled = false;
+		$asyncJobBackendError = '';
+		try {
+			$isAsyncJobBackendInstalled = Router::getRoute('AsyncJobs')->isInstalled();
+		} catch (Exception $e) {
+			$asyncJobBackendError = $e->getMessage();
+		}
+
 		UI::table([
 			'caption' => UI::h('b', ['style' => "color:#000"], [
 				"Baza danych",
@@ -88,7 +92,20 @@ class Route_Status extends RouteBase {
 			]),
 			'rows' => [
 				[
-					'nazwa' => "Event Scheduler (generowanie Grafika, itp.)",
+					'nazwa' => "Async Job backend (WPS Serwer)",
+					'wartość' => ($isAsyncJobBackendInstalled)
+					?	UI::h('span', ['class' => "label label-success"], "ON")
+					:	UI::h('span', [
+						'class' => "label label-danger",
+						'title' => $asyncJobBackendError,
+					], "OFF"),
+					'#' => UI::h('a', [
+						'class' => "btn btn-xs btn-link",
+						'href' => Router::getRoute('AsyncJobs')->getLink(),
+					], "info")
+				],
+				[
+					'nazwa' => "DB / Event Scheduler (generowanie Grafika, itp.)",
 					'wartość' => ('ON' == $dbEvents['Value'])
 					?	UI::h('span', ['class' => "label label-success"], "ON")
 					:	UI::h('span', ['class' => "label label-danger"], "OFF"),
@@ -100,7 +117,7 @@ class Route_Status extends RouteBase {
 					])
 				],
 				[
-					'nazwa' => "System obiektów (xsd)",
+					'nazwa' => "DB / System obiektów (xsd)",
 					'wartość' => ($aclObjectCacheExists)
 					?	UI::h('span', ['class' => "label label-success"], "ON")
 					:	UI::h('span', ['class' => "label label-danger"], "OFF"),
@@ -112,7 +129,7 @@ class Route_Status extends RouteBase {
 					])
 				],
 				[
-					'nazwa' => "Tabela z relacjami (REF)",
+					'nazwa' => "DB / Tabela z relacjami (REF)",
 					'wartość' => ($totalToUpdateRef)
 					?	UI::h('span', ['class' => "label label-danger"], "wymaga aktualizacji ({$totalToUpdateRef})")
 					:	UI::h('span', ['class' => "label label-success"], "akualne"),

SE/se-lib/Route/Storage.php

@@ -67,7 +67,7 @@ class Route_Storage extends RouteBase {
 						[
 							'Nr zasobu' => '',
 							'nazwa' => "Narzędzia systemowe",
-							'opis' => "SystemObjects",
+							'opis' => "Narzędzia systemowe",
 							'config?' => '<span class="text text-muted">n/d</span>',
 							'obiekty' => UI::h('a', [ 'href' => Router::getRoute('Storage_Tools')->getLink() ], "narzędzia"),
 						],

SE/se-lib/Route/Storage/Tools.php

@@ -12,8 +12,11 @@ class Route_Storage_Tools extends RouteBase {
 	}
 	public function defaultView() {
 		echo $this->getLink();
+
 		throw new Exception("TODO...");
 		// TODO: @see RouteBase::reinstall()
+		// TODO: use Schema_SystemFunctionStorageAcl to fetch list
+		// TODO: reload list: fetch all from files again
 		// TODO: SE/projects/{activeProject}/tools/*.php
 	}
 

SE/se-lib/Route/UrlAction/Ant.php.async.js

@@ -0,0 +1,90 @@
+if (!RUN_ANT_JOB_URL) throw "Missing RUN_ANT_JOB_URL";
+if (!RUN_TEST_ANT_JOB_URL) throw "Missing RUN_TEST_ANT_JOB_URL";
+
+var DBG = (DBG) ? true : false;
+var DBG1 = true;
+
+
+function p5UI_runAntAsyncJob(event, targetNode, props) {
+	event.stopPropagation();
+	event.preventDefault();
+	var props = props || {};
+
+	var href = targetNode.getAttribute('href');
+
+	var namespace = targetNode.getAttribute('data-namespace');
+	var primaryKey = targetNode.getAttribute('data-primaryKey');
+	var ant_path = targetNode.getAttribute('data-ant_path');
+	var ant_template = targetNode.getAttribute('data-ant_template');
+
+	DBG1 && console.log("DBG:p5UI_runAntAsyncJob ", { props_TEST: props.TEST ? true : false, targetNode, href, namespace, primaryKey, ant_path, ant_template })
+
+	if (targetNode._isLoading) return;
+
+	{
+		targetNode._isLoading = true;
+		var handleRemoveRefreshIcon = _addRefreshIcon(targetNode);
+	
+		var handleRemoveRefreshIcon = (function (targetNode) {
+			return function () {
+				if (targetNode._isLoading) {
+					targetNode._isLoading = false;
+					_removeRefreshIcon(targetNode);
+				}
+			}
+		})(targetNode);
+
+		// setTimeout(handleRemoveRefreshIcon, 1000);
+		global.fetch(props.TEST ? RUN_TEST_ANT_JOB_URL : RUN_ANT_JOB_URL, {
+			method: 'POST',
+			header: {
+				'contentType': 'applications/json'
+			},
+			credentials: 'same-origin',
+			body: JSON.stringify({
+				namespace,
+				primaryKey,
+				ant_path,
+				ant_template,
+			})
+		}).then(function (response) {
+			handleRemoveRefreshIcon()
+			return response.json()
+		})
+		.then(function (result) {
+			DBG1 && console.log("DBG:response", { result })
+			if (result.type && 'error' === result.type) return _handleResponseError(result);
+			if (result.type && 'success' === result.type) return _handleResponseSuccess(result);
+			_handleResponseError(result);
+		});
+
+	}
+
+	return false;
+}
+
+function _handleResponseError(result) {
+	p5UI__notifyAjaxCallback(result);
+}
+function _handleResponseSuccess(result) {
+	p5UI__notifyAjaxCallback(result);
+}
+
+function _addRefreshIcon(targetNode) {
+	var loadingNode = document.createElement('i');
+	loadingNode.setAttribute('class', 'glyphicon glyphicon-refresh');
+	loadingNode.style.marginLeft = "8px";
+	loadingNode._p5Node = 'loading-icon';
+	targetNode.appendChild(loadingNode);
+}
+function _removeRefreshIcon(targetNode) {
+	var lastNode = targetNode.lastChild;
+	if (!lastNode) return;
+	DBG1 && console.log("DBG:_removeRefreshIcon", { targetNode, lastNode, lastNode_p5Node: lastNode._p5Node });
+	if (!lastNode._p5Node) return;
+	if ('loading-icon' != lastNode._p5Node) return;
+	targetNode.removeChild(lastNode);
+}
+
+
+global.p5UI_runAntAsyncJob = p5UI_runAntAsyncJob;

SE/se-lib/Schema/SystemFunctionStorageAcl.php

@@ -63,6 +63,7 @@ class Schema_SystemFunctionStorageAcl extends Core_AclSimpleSchemaBase {
 		if (null !== $_cacheAllItems) return $_cacheAllItems;
 
 		// TODO: read from filesystem
+		// TODO: - read from SE/schema/ant-url_action
 		// TODO: set ON / OFF in companies settings or remote repo for company tools
 		// TODO: install Tool into CRM_LISTA_ZASOBOW - auto
 		// TODO: panel for connect tool with table - Storage_Tools