BiAuditGenerate - dopieszczanie logów

SE/se-lib/Route/UrlAction/BiAuditGenerate.php

@@ -1135,6 +1135,9 @@ Router::getRoute("UrlAction_BiAuditGenerate")->' . $function . ';
 					$generatePowiazaniaResultFile = "{$tasksDirLocation}/generatePowiazania-{$row['ID']}.result";
 					$generatePowiazaniaPhpFile = "{$tasksDirLocation}/generatePowiazania-{$row['ID']}.php";
 					$generatePowiazaniaLogFile = "{$tasksDirLocation}/generatePowiazania-{$row['ID']}.log";
+					$generatePowiazaniaProgressFile = "{$tasksDirLocation}/generatePowiazania-{$row['ID']}.progress";
+					if (file_exists($generatePowiazaniaResultFile)) unlink ($generatePowiazaniaResultFile);
+					if (file_exists($generatePowiazaniaProgressFile)) unlink ($generatePowiazaniaProgressFile);
 					file_put_contents($generatePowiazaniaPhpFile, $generatePhpScript("doGeneratePowiazania({$row['ID']})"));
 					shell_exec('su - root -c "php ' . $generatePowiazaniaPhpFile . ' > ' . $generatePowiazaniaLogFile. ' 2>&1 &"');
 				}
@@ -1982,11 +1985,15 @@ class BiAuditPowiazania {
 	private $step = 0;
 	private $tasksDirLocation;
 	private $progressFile;
+	private $progress = [];
 	private $lastProgress = 0;
+	private $lastResults = 0;
 	private $startTimestamp = null;
+	private $lastTimestamp = null;
 	private $lowProgressCount = 0;
 	private $count, $i;
 	private $splittedResults = [];
+	private $maxResults = 200000;
 
 	private function getRefTable($tableFrom, $tableTo, $returnException = true) {
 		if (!isset($this->refTables[$tableFrom][$tableTo])) {
@@ -2009,16 +2016,34 @@ class BiAuditPowiazania {
 		echo $messageJson . "\n";
 	}
 
-	private function saveProgress($progress) {
-		$this->lastProgress = $progress;
+	private static function exec($cmd) {
+		echo "CMD :: {$cmd}\n";
+		shell_exec($cmd);
+	}
+
+	private function saveProgress($progress, $message = null) {
 		$timestamp = microtime(true);
-		if ($progress) $estimated = ($timestamp - $this->startTimestamp) / $progress;
-		else $estimated = "N/A";
-		$progressJson = self::generateJson('progress', ['progress' => $progress, 'item' => ($this->i + 1) . "/" . $this->count, 'results' => count($this->results), 'step' => $this->step, 'estimated' => $estimated]);
-		echo $progressJson . "\n";
-		file_put_contents($this->progressFile, $progressJson);
+		if ($progress !== null && $this->i) {
+			$this->lastProgress = $progress;
+			$elapsed = $timestamp - $this->startTimestamp;
+			if ($progress) {
+				$lastElapsed = $timestamp - $this->lastTimestamp;
+				$lastProgress = $progress - $this->lastProgress;
+				$estimated = (($elapsed * (1 - $progress)) / $progress + ($lastElapsed * (1 - $lastProgress)) / $lastProgress) / 2;
+			} else $estimated = "N/A";
+			$this->lastProgress = $progress;
+			$this->lastTimestamp = $timestamp;
+			$progressJson = self::generateJson('progress', ['progress' => $progress, 'item' => $this->i . "/" . $this->count, 'results' => count($this->results), 'step' => $this->step, 'elapsed' => $elapsed, 'estimated' => $estimated]);
+			echo $progressJson . "\n";
+			$this->progress[$this->i] = ['progress' => $progress, 'results' => (count($this->results) - $this->lastResults), 'elapsed' => $elapsed, 'estimated' => $estimated];
+		}
+		if ($message !== null) $this->progress[0]['message'] = $message;
+		$this->progress[0]['ts'] = $timestamp;
+		file_put_contents($this->progressFile, json_encode($this->progress));
 	}
 
+	
+
 	private static function throwException($message) {
 		self::saveToLog($message);
 		throw new Exception($message);
@@ -2065,32 +2090,54 @@ class BiAuditPowiazania {
 			on `all`.REMOTE_TABLE = 'BI_audit_ENERGA_PRACOWNICY' and `all`.REMOTE_ID = ref.REMOTE_PRIMARY_KEY and ref.PRIMARY_KEY = '{$this->ID}' order by `all`.`ID` asc";
 		$result = DB::getPDO()->fetchAll($query);
 		$this->count = count($result);
-		$this->saveProgress(0);
+		$this->progress[0]['count'] = $this->count;
 		$this->origMaxDepth = $this->maxDepth;
-		foreach ($result as $this->i => $row) {
-			if ($this->maxDepth != $this->origMaxDepth) {
-				$this->maxDepth = $this->origMaxDepth;
-				self::saveToLog("Przywrócono oryginalną maksymalną głębokość przeszukiwania ({$this->origMaxDepth})");
+
+		try {
+			foreach ($result as $i => $row) {
+				$this->i = $i + 1;
+				$this->startTimestamp = microtime(true);
+				$this->lastProgress = 0;
+				$this->saveProgress(0, "Wyszukiwanie powiązań dla obiektu {$this->i}/{$this->count}");
+				if ($this->maxDepth != $this->origMaxDepth) {
+					$this->maxDepth = $this->origMaxDepth;
+					self::saveToLog("Przywrócono oryginalną maksymalną głębokość przeszukiwania ({$this->origMaxDepth})");
+				}
+				$this->findPowiazania($row['ID']);
+				$this->saveProgress(1, "Zakończono wyszukiwanie powiązań dla obiektu {$this->i}/{$this->count}");
+				$this->lastResults = count($this->results);
 			}
-			$this->findPowiazania($row['ID']);
-			$this->saveProgress(1);
+		} catch (Exception $e) {
+			if ($this->i < $this->count) {
+				$this->startTimestamp = microtime(true);
+				$this->lastProgress = 0;
+				$this->lastResults = count($this->results);
+				while (++$this->i <= $this->count) $this->saveProgress(0);
+			}
+			if ($message = $e->getMessage()) self::saveToLog("Wystąpił błąd podczas wyszukiwania powiązań - {$message}");
 		}
+
 		$this->nodes = [];
 		self::saveToLog("Zakończono wyszukiwanie powiązań");
 		$this->saveResults();
-		$this->saveToDb();
+		//$this->saveToDb(); 2017-12-14 - przerobiono na zapisywanie danych do bazy w locie
 		$this->splitResults();
 	}
 
-	public function findPowiazania($ID, $progress = 0, $steps = 1, $relation = "", $end = 0) {
-		if (!$progress) {
-			$this->startTimestamp = microtime(true);
-			$this->lastProgress = 0;
+	private function addResult($progress) {
+		if (count($this->results) == $this->maxResults) {
+			$message = "Przerwano wyszukiwanie powiązań - osiągnięto maksymalną liczbę wyników {$this->maxResults}";
+			self::saveToLog($message);
+			$this->saveProgress($progress, $message);
+			throw new Exception('');
 		}
+		$this->results[] = $this->path;
+		$this->saveToDb($this->path);           
+	}
 
+	public function findPowiazania($ID, $progress = 0, $steps = 1, $relation = "", $end = 0) {
 		if (((++$this->step) % 1000000) == 0) {
 			$progressDiff = $progress - $this->lastProgress;
-//			self::saveToLog("TEST: " . (number_format($progress - $this->lastProgress, 20)));
 			if ($progressDiff < 0.00005) {
 				$this->lowProgressCount++;
 				if ($this->lowProgressCount == 10) {
@@ -2099,9 +2146,12 @@ class BiAuditPowiazania {
 				}
 			} else $this->lowProgressCount = 0;
 			$this->saveProgress($progress);
-			if (memory_get_usage(true) > 4000000000) {
+		}
+
+		if ((($this->step) % 10000) == 0) {
+			if (memory_get_usage(true) > 3000000000) {
 				$this->nodes = [];
-				self::saveToLog("Too much memory used, cleaning relations cache");
+				self::saveToLog("Oczyszczam podręczny cache - zajmuje zbyt dużo pamięci");
 			}
 		}
 
@@ -2111,13 +2161,13 @@ class BiAuditPowiazania {
 		if (!$relation) $relation = 0;
 
 		if ($end) {
-			if (in_array($ID, $this->endNodes) && count($this->path) > $this->minDepth) $this->results[] = $this->path;
+			if (in_array($ID, $this->endNodes) && count($this->path) > $this->minDepth) $this->addResult($progress);
 			array_pop($this->path);
 			return;
 		}
 
 		if (count($this->path) > $this->maxDepth) {
-			if (!$this->onlyTargets) $this->results[] = $this->path;
+			if (!$this->onlyTargets) $this->addResult($progress);
 			array_pop($this->path);
 			return;
 		}
@@ -2286,15 +2336,18 @@ class BiAuditPowiazania {
 		self::saveToLog("Wygenerowano dane na potrzeby utworzenia pliku XML ({$resultsPart})");
 	}
 
-	private function saveToDb() {
-		self::saveToLog("Zapisuję wyliczone dane do bazy");
-		if (!$this->results) {
-			self::saveToLog("Brak wyliczonych danych - nie zapisaono niczego do bazy");
-			return null;
-		}
-		$refPowiazaniaToPowiazaniaRow = $this->getRefTable('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA', 'BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row');
-		$refPowiazaniaRowToPowiazaniaRowObject = $this->getRefTable('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row', 'BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row_object');
-		foreach ($this->results as $result) {
+	private function saveToDb($result = null) {
+		if ($result === null) {
+			self::saveToLog("Zapisuję wyliczone dane do bazy");
+			if (!$this->results) {
+				self::saveToLog("Brak wyliczonych danych - nie zapisaono niczego do bazy");
+				return null;
+			}
+			foreach ($this->results as $result) $this->saveToDb($result);
+			self::saveToLog("Zapisano wyliczone dane do bazy");
+		} else {
+			$refPowiazaniaToPowiazaniaRow = $this->getRefTable('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA', 'BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row');
+			$refPowiazaniaRowToPowiazaniaRowObject = $this->getRefTable('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row', 'BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row_object');
 			$IdPowiazaniaRow = DB::getPDO()->insert('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA_row', []);
 			DB::getPDO()->insert($refPowiazaniaToPowiazaniaRow, ['PRIMARY_KEY' => $this->ID, 'REMOTE_PRIMARY_KEY' => $IdPowiazaniaRow]);
 			foreach ($result as $ID => $rel) {
@@ -2306,7 +2359,6 @@ class BiAuditPowiazania {
 				DB::getPDO()->insert($ref, ['PRIMARY_KEY' => $idPowiazaniaRowObject, 'REMOTE_PRIMARY_KEY' => $object['REMOTE_ID']]);
 			}
 		}
-		self::saveToLog("Zapisano wyliczone dane do bazy");
 	}
 
 	public function saveXml() {
@@ -2383,19 +2435,19 @@ class BiAuditPowiazania {
 			self::saveToLog("Nie wygenerowano plików PDF oraz HTML - problem z utworzeniem katalogu dla ant ({$resultsPart})");
 		} else {
 			copy($xmlFile, $antXmlFile);
-			shell_exec("cd /Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree && /opt/local/bin/ant -f build_CRM_PROCES_tree.xml default_db:PROCES_INIT:tree:dita -Duuid=relations-{$id_part}");
+			self::exec("cd /Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree && /opt/local/bin/ant -f build_CRM_PROCES_tree.xml default_db:PROCES_INIT:tree:dita -Duuid=relations-{$id_part}");
 			self::saveToLog("Generuję plik PDF ({$resultsPart})");
-			shell_exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && /Library/Server/Web/Data/Sites/Default/SE/stuff/dita-ot-2.3.3/bin/dita -o pdf -i relations-{$id_part}.ditamap -f pdf");
+			self::exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && /Library/Server/Web/Data/Sites/Default/SE/stuff/dita-ot-2.3.3/bin/dita -o pdf -i relations-{$id_part}.ditamap -f pdf");
 			if (file_exists($pdfFile)) {
 				rename($pdfFile, $pdfDestFile);
 				self::saveToLog("Wygenerowano plik PDF ({$resultsPart})");
-			} else $self::saveToLog("Nie udało się wygenerować pliku PDF ({$resultsPart})");
+			} else self::saveToLog("Nie udało się wygenerować pliku PDF ({$resultsPart})");
 			self::saveToLog("Generuję pliki HTML ({$resultsPart})");
-			shell_exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && /Library/Server/Web/Data/Sites/Default/SE/stuff/dita-ot-2.3.3/bin/dita -o html -i relations-{$id_part}.ditamap -f tocjs");
+			self::exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && /Library/Server/Web/Data/Sites/Default/SE/stuff/dita-ot-2.3.3/bin/dita -o html -i relations-{$id_part}.ditamap -f tocjs");
 			if (file_exists($htmlDir) && is_dir($htmlDir)) {
-				shell_exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && zip -r \"{$htmlZipDestFile}\" html");
-				if (file_exists($htmlDestDir)) shell_exec("rm -rf \"{$htmlDestDir}\"");
-				shell_exec("mv \"{$htmlDir}\" \"{$htmlDestDir}\"");
+				self::exec("cd \"/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/CRM_PROCES_tree/temp/relations-{$id_part}\" && zip -r \"{$htmlZipDestFile}\" html");
+				if (file_exists($htmlDestDir)) self::exec("rm -rf \"{$htmlDestDir}\"");
+				self::exec("mv \"{$htmlDir}\" \"{$htmlDestDir}\"");
 				self::saveToLog("Wygenerowano pliki HTML ({$resultsPart})");
 			} else self::saveToLog("Nie udało się wygenerować plików HTML {$resultsPart})");
 		}