se/SE/se-lib/Core/AsyncJobs.php

<?php

// /opt/local/pl.procesy5/p5build_SE/temp/WPS_Functions/default_db/CRM_PROCES_tree/relations-22
// /opt/local/pl.procesy5/ -> /Library/New_Server/opt/local/pl.procesy5
// /opt/local/pl.procesy5/async_jobs - TODO: APP_PATH_ASYNC_JOB from .htaccess or config

// Config example:
// cat SE/config/.cnf-biuro.biall-net.pl.ini.php:
// APP_PATH_ASYNC_JOB="/opt/local/pl.procesy5/async_jobs"
// register new job:
// - $TODAY = date("Y-m-d");
// - $JOB_ID = generateJobID if not already started?
// - make folder APP_PATH_ASYNC_JOB / $TODAY / job-{$JOB_ID}
// - make base files:
//   - log files: `out.log`, `error.log`
//   - `output`: if process creates file. @param --out_file=out
//   - `output_type`: output mime type or namespace
//   - `progress`: if process implement progress. @param --progress_file=progress

Lib::loadClass('Request');
Lib::loadClass('Core_AsyncJobsFiles');
Lib::loadClass('Core_AsyncJobsDB');
Lib::loadClass('Core_AsyncJobsServer'); // pm2

/**
 * @usage: Core_AsyncJobs::startAsyncJob($idJob)
 *      -> Core_AsyncJobsServer::startAsyncJob(execPath, name, [ 'out.log', 'error.log', 'cwd', 'args' ])
 */

class Core_AsyncJobs {

	static function registerNewAntTask($props) {
		$namespace = V::get('namespace', '', $props);
		$primaryKey = V::get('primaryKey', '', $props);
		$ant_path = V::get('ant_path', '', $props);
		$ant_template = V::get('ant_template', '', $props);

		if (empty($namespace)) throw new Exception("Missing namespace");
		if (empty($primaryKey)) throw new Exception("Missing primaryKey");
		if (empty($ant_path)) throw new Exception("Missing ant_path");
		if (empty($ant_template)) throw new Exception("Missing ant_template");

		// https://biuro.biall-net.pl/dev-pl/se-master/index.php?_route=UrlAction_Ant
		// & _task=ant
		// & path=default_db.in7_dziennik_koresp/test-async
		// & template=test-loop
		// & typeName=default_db:IN7_DZIENNIK_KORESP
		// & primaryKey=66263
		// & primaryKeyField=ID

		Lib::loadClass('ACL');
		$acl = ACL::getAclByNamespace($namespace);
		$typeName = Api_WfsNs::typeName($namespace);
		$pkField = $acl->getPrimaryKeyField();

		// DB::getPDO()->tryHandleException
		$idJob = V::tryHandleException($handler = [ 'Core_AsyncJobs', 'isInstalled' ], $callback = [ 'Core_AsyncJobsDB', 'insert' ], $args = [
			[
				'JOB_NAME' => "Ant|{$namespace}|{$ant_path}|{$ant_template}",
				'LOCK_VALUE' => $primaryKey,
				'USER' => User::getLogin(),
			],
		]);

		{
			$path = $ant_path; // default_db.in7_dziennik_koresp/test-async
			$template = $ant_template; // test-loop

			$webRootUrl = Request::getPathUri() . "schema/ant-url_action/{$path}"; // TODO: security - only for test
			$outputFunctionUrl = Router::getRoute('UrlAction_Ant')->getLink('output') . "&path={$path}&file=";
			$antFunctionUrl = Router::getRoute('UrlAction_Ant')->getLink('ant') . "&path={$path}&file={$file}&template={$template}&typeName={$typeName}&primaryKey={$primaryKey}&primaryKeyField={$pkField}"; //  &confirmAntfile={$confirmAntfile}&confirmAntfileTarget={$confirmAntfileTarget} sdo confirmacji potrzebne - wzglednie przetwarzac z tresci output URL_TASK - ale potrzebne parametry analogiczne aby chodzily
			// $testUrl = Request::getPathUri() . "wfs-data.php/default_db/?SERVICE=WFS&VERSION=1.0.0&TYPENAME={$typeName}&SRSNAME=EPSG:3003&featureID={$objectName}.{$primaryKey}";// &REQUEST=GetFeature
			$uuid = date("Y-m-d-H_i_s") . substr(md5(time()), 0, 6); // TODO: uniq id for every request
			Lib::loadClass('Crypt');
			$passwordBase64Basic = base64_encode(User::getLogin() . ":" . Crypt::decrypt($_SESSION['ADM_PASS_HASH']));
			// /* TODO: ??? */ if( strlen($confirmAntfile) > 0 ) $cmd .= " -DconfirmAntfile={$confirmAntfile}";
			// /* TODO: ??? */ if( strlen($confirmAntfileTarget) > 0 ) $cmd .= " -DconfirmAntfileTarget={$confirmAntfileTarget}";
			$php_session_id = session_id();
		}
		Core_AsyncJobsFiles::createNewJobFiles($idJob, $jobProps = [
			'webRootUrl' => $webRootUrl,
			'outputFunctionUrl' => $outputFunctionUrl,
			'antFunctionUrl' => $antFunctionUrl,
			'uuid' => $uuid,
			'passwordBase64Basic' => $passwordBase64Basic,
			'php_session_id' => $passwordBase64Basic,
			'xpath' => $pkField,
			'xpath_value' => $primaryKey,
			'template' => $template,
			'api_url' => Request::getPathUri()."wfs-data.php", //potrzebuje ten parametr do dzialania w AMS itp
			'typeName' => $typeName,
		]);

		$jobStartScript = "";
		{
			$propFilePath = Core_AsyncJobsFiles::propertyFile($idJob, $today = date("Y-m-d"));
			$cmd = "cd " . APP_PATH_SCHEMA . "/ant-url_action/{$ant_path}";
			$cmd .= " && ";
			$cmd .= implode(" ", [
				'ant',
				'-S',
				'-propertyfile', $propFilePath,
				($ant_template) ? $ant_template : '',
				'2>&1',
			]);

			$pathCmd = [];
			$pathCmd[] = '/opt/local/bin/ant';
			$pathCmd[] = '/bin';
			$pathCmd[] = '/usr/bin';
			$pathCmd[] = '/usr/local/bin';
			$pathCmd[] = '/opt/local/bin';
			$pathCmd[] = '/sbin';
			$pathCmd[] = '/usr/sbin';
			$pathCmd[] = '/opt/local/sbin/skrypty';
			$pathCmd[] = '/opt/local/var/macports/software';
			$pathCmd[] = '/Applications/Server.app/Contents/ServerRoot/usr/bin';
			$pathCmd[] = '/Applications/Server.app/Contents/ServerRoot/usr/sbin';
			$jobStartScript = implode("\n", [
				'export PATH=' . implode(':', $pathCmd),
				'JAVA_HOME="/usr/bin/java"',
				'MAVEN_OPTS="-Xms256m -Xmx512m"',
				$cmd,
				"",
			]);
		}
		Core_AsyncJobsFiles::createTaskStartScript($idJob, $jobDate = date("Y-m-d"), $jobStartScript);

		return $idJob;
	}

	static function startAsyncJob($idJob, $item = []) {
		if (empty($item)) $item = Core_AsyncJobsDB::fetch($idJob);
		DBG::nicePrint($item, '$item');
		// Core_AsyncJobsFiles::basePath($idJob, $jobDate = $item['DATE']);
		// Core_AsyncJobsFiles::propertyFile($idJob, $jobDate = $item['DATE']);
		// Core_AsyncJobsFiles::startScript($idJob, $jobDate = $item['DATE']);
		// Core_AsyncJobsFiles::outLog($idJob, $jobDate = $item['DATE']);
		// Core_AsyncJobsFiles::errorLog($idJob, $jobDate = $item['DATE']);
		$jobDate = $item['DATE'];
		$execPath = Core_AsyncJobsFiles::startScript($idJob, $jobDate);

		// Core_AsyncJobsServer::startAsyncJob(execPath, name, [ 'out.log', 'error.log', 'cwd', 'args' ])
		$out = [];
		$ret = Core_AsyncJobsServer::startAsyncJob(
			$execPath,
			$name = implode("|", [ $item['JOB_NAME'], $item['LOCK_VALUE'] ]),
			[
				'out.log' => Core_AsyncJobsFiles::outLog($idJob, $jobDate),
				'error.log' => Core_AsyncJobsFiles::errorLog($idJob, $jobDate),
				'cwd' => dirname(Core_AsyncJobsFiles::basePath($idJob, $jobDate)),
			],
			$out
		);
		if (0 === $ret) {
			throw new AlertInfoException("Uruchomiono zadanie. " . implode(" ", $out));
		} else {
			throw new Exception("Nie uruchomiono zadania. " . implode(" ", $out));
		}
	}

	static function getSimpleList() {
		$fullList = self::getFullList();
		return array_map(function ($jobInfo) {
			return [
				'name' => $jobInfo['name'],
				'pid' => $jobInfo['pid'],
				'pm_id' => $jobInfo['pm_id'],
				'pm2_env.status' => $jobInfo['pm2_env']['status'],
				'monit.memory' => $jobInfo['monit']['memory'],
				'monit.cpu' => $jobInfo['monit']['cpu'],
				//   "pm_out_log_path": "/Library/WebServer/.pm2/logs/test-job-1-out.log",
				//   "pm_err_log_path": "/Library/WebServer/.pm2/logs/test-job-1-error.log",
				//   "pm_pid_path": "/Library/WebServer/.pm2/pids/test-job-0.pid",
				'pm2_env.pm_out_log_path' => $jobInfo['pm2_env']['pm_out_log_path'],
				'pm2_env.pm_err_log_path' => $jobInfo['pm2_env']['pm_err_log_path'],
				'pm2_env.pm_pid_path' => $jobInfo['pm2_env']['pm_pid_path'],
			];
		}, $fullList);
	}

	static function getFullList() {
		$jobListJson = V::shell_exec("pm2 jlist 2>&1");
		if (empty($jobListJson)) throw new Exception("Reading async job list failed");
		$parsedJobList = @json_decode($jobListJson, $assoc = true);
		if (null == $parsedJobList && 0 !== json_last_error()) throw new Exception("Parsing async job list failed: " . json_last_error());
		return $parsedJobList;
	}

	static function getJobLogs($jobNameOrID, $lastLines = 10) {
		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in getJobLogs");
		$lastLines = ((int)$lastLines > 0) ? (int)$lastLines : 10;
		// $cmd = "pm2 logs {$jobNameOrID} --lines {$lastLines} --nostream | tail -n {$lastLines}";
		$cmd = "pm2 logs {$jobNameOrID} --lines {$lastLines} --nostream";
		return V::shell_exec($cmd);
		// [TAILING] Tailing last 3 lines for [0] process (change the value with --lines option)
		// /Library/WebServer/.pm2/logs/test-job-1-out.log last 3 lines:
		// /Library/WebServer/
	}

	static function stopJob($jobNameOrID) {
		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in stopJob");
		$cmd = "pm2 stop {$jobNameOrID}";
		return V::shell_exec($cmd);
	}

	static function startJob($jobNameOrID) {
		if (empty($jobNameOrID) && $jobNameOrID !== 0) throw new Exception("Missing job name or id in startJob");
		$cmd = "pm2 start {$jobNameOrID}";
		return V::shell_exec($cmd);
	}

	static function deleteStopped() {
		// TODO: script to remove stopped in loop with delay
		// pm2 start app.js --restart-delay=3000
		// $ pm2 list | grep '^│' |  awk -F'│' '{ gsub(/ /, "", $2); gsub(/ /, "", $10); if ("stopped" == $10) { print $2" # STOPPED" } else { print $10 } }' |  grep '# STOPPED' | xargs -n1 pm2 delete
		$testCmd = implode(" | ", [
			"pm2 list",
			"grep '^│'",
			"awk -F'│' '{ gsub(/ /, \"\", \$2); gsub(/ /, \"\", \$10); if (\"stopped\" == \$10) { print \$2\" # STOPPED\" } else { print \$10 } }'",
			"grep '# STOPPED'",
			"awk '{print \$1}'",
		]);
		V::exec($testCmd . " 2>&1", $out, $ret);
		echo "cmd: <code>{$cmd}</code><br>RETURN CODE: '{$ret}'<br><pre>OUTPUT:\n" . implode("\n", $out) . "</pre>";

		$cmd = implode(" | ", [
			"pm2 list",
			"grep '^│'",
			"awk -F'│' '{ gsub(/ /, \"\", \$2); gsub(/ /, \"\", \$10); if (\"stopped\" == \$10) { print \$2\" # STOPPED\" } else { print \$10 } }'",
			"grep '# STOPPED'",
			"awk '{print \$1}'",
			"xargs -n1 pm2 delete",
		]);
	}

	static function isInstalled() {
		if (!Core_AsyncJobsFiles::isInstalled()) throw new Exception("AsyncJobs files not installed");

		if (!Core_AsyncJobsServer::isInstalled()) throw new Exception("AsyncJobs server not installed");

		if (!Core_AsyncJobsDB::checkAsyncJobDatabase()) throw new Exception("AsyncJobs database schema not installed");

		return true;
	}

	// pm2 logs -h
	//
	//   Usage: logs [options] [id|name]
	//   stream logs file. Default stream all logs
	//   Options:
	//     --json                json log output
	//     --format              formated log output
	//     --raw                 raw output
	//     --err                 only shows error output
	//     --out                 only shows standard output
	//     --lines <n>           output the last N lines, instead of the last 15 by default
	//     --timestamp [format]  add timestamps (default format YYYY-MM-DD-HH:mm:ss)
	//     --nostream            print logs without lauching the log stream
	//     --highlight [value]   highlights the given value
	//     -h, --help            output usage information


	// [0] => Array:
	//     [pid] => 71728
	//     [name] => test-job-1
	//     [pm2_env] => Array:
	//         [exit_code] => 0
	//         [versioning] => 
	//         [version] => N/A
	//         [unstable_restarts] => 0
	//         [restart_time] => 53
	//         [pm_id] => 0
	//         [created_at] => 1581938950672
	//         [axm_dynamic] => Array:
	//         [axm_options] => Array:
	//         [axm_monitor] => Array:
	//         [axm_actions] => Array:
	//         [pm_uptime] => 1581939524854
	//         [status] => online
	//         [unique_id] => 80fefcff-8605-44cc-9829-bbcba4de4e7c
	//         [PM2_HOME] => /Library/WebServer/.pm2
	//         [PATH] => /usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/local/bin:/opt/local/lib/mysql55/bin:/Applications/Server.app/Contents/ServerRoot/usr/bin:/Applications/Server.app/Contents/ServerRoot/usr/sbin:/Users/pl/programy/bin
	//         [PWD] => /Users/plabudda/rsync.se.master/SE
	//         [XPC_FLAGS] => 0x80
	//         [XPC_SERVICE_NAME] => 0
	//         [HOME] => /Library/WebServer
	//         [SHLVL] => 1
	//         [SERVER_INSTALL_PATH_PREFIX] => /Applications/Server.app/Contents/ServerRoot
	//         [XPC_SERVICES_UNAVAILABLE] => 1
	//         [MODULE_INSTALL_PATH_PREFIX] => 
	//         [_] => /usr/local/bin/pm2
	//         [__CF_USER_TEXT_ENCODING] => 0x46:0:0
	//         [PM2_USAGE] => CLI
	//         [NODE_APP_INSTANCE] => 0
	//         [vizion_running] => 
	//         [km_link] => 
	//         [pm_pid_path] => /Library/WebServer/.pm2/pids/test-job-0.pid
	//         [pm_err_log_path] => /Library/WebServer/.pm2/logs/test-job-1-error.log
	//         [pm_out_log_path] => /Library/WebServer/.pm2/logs/test-job-1-out.log
	//         [instances] => 1
	//         [exec_mode] => fork_mode
	//         [exec_interpreter] => php
	//         [pm_cwd] => /Users/plabudda/rsync.se.master/SE
	//         [pm_exec_path] => /Users/plabudda/rsync.se.master/sbin/test-sleep-loop.php
	//         [node_args] => Array:
	//         [name] => test-job-1
	//         [namespace] => default
	//         [env] => Array:
	//             [unique_id] => 80fefcff-8605-44cc-9829-bbcba4de4e7c
	//             [test-job-1] => Array:
	//             [PM2_HOME] => /Library/WebServer/.pm2
	//             [PATH] => /usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/local/bin:/opt/local/lib/mysql55/bin:/Applications/Server.app/Contents/ServerRoot/usr/bin:/Applications/Server.app/Contents/ServerRoot/usr/sbin:/Users/pl/programy/bin
	//             [PWD] => /Users/plabudda/rsync.se.master/SE
	//             [XPC_FLAGS] => 0x80
	//             [XPC_SERVICE_NAME] => 0
	//             [HOME] => /Library/WebServer
	//             [SHLVL] => 1
	//             [SERVER_INSTALL_PATH_PREFIX] => /Applications/Server.app/Contents/ServerRoot
	//             [XPC_SERVICES_UNAVAILABLE] => 1
	//             [MODULE_INSTALL_PATH_PREFIX] => 
	//             [_] => /usr/local/bin/pm2
	//             [__CF_USER_TEXT_ENCODING] => 0x46:0:0
	//             [PM2_USAGE] => CLI
	//         [merge_logs] => 1
	//         [vizion] => 1
	//         [autorestart] => 1
	//         [watch] => 
	//         [instance_var] => NODE_APP_INSTANCE
	//         [pmx] => 1
	//         [automation] => 1
	//         [treekill] => 1
	//         [username] => _www
	//         [windowsHide] => 1
	//         [kill_retry_time] => 100
	//     [pm_id] => 0
	//     [monit] => Array:
	//         [memory] => 15720448
	//         [cpu] => 0

}