BiAuditGenerate - logi, automatyczne zmniejszanie maxDepth

SE/se-lib/Route/UrlAction/BiAuditGenerate.php

@@ -1039,7 +1039,7 @@ Router::getRoute("UrlAction_BiAuditGenerate")->' . $function . ';
 			$query = "select ID from BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA where BI_analiza_reloadCache not in ('Full', 'Part') and FILE_STATUS = 'TO_GENERATE'";
 			$result = DB::getPDO()->fetchAll($query);
 			foreach ($result as $row) {
-				DB::getDB()->UPDATE_OBJ('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA', array_merge(['ID' => $row['ID']], $sqlArr));
+// DUPA				DB::getDB()->UPDATE_OBJ('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA', array_merge(['ID' => $row['ID']], $sqlArr));
 			}
 
 			### Sprawdzenie czy któryś rekord wymaga odświeżenia cache
@@ -1777,11 +1777,11 @@ Router::getRoute("UrlAction_BiAuditGenerate")->' . $function . ';
 		];
 
 		try {
-			$BiAuditPowiazania = new BiAuditPowiazania($ID);
+			$BiAuditPowiazania = new BiAuditPowiazania($ID, $tasksDirLocation);
 			if (!$BiAuditPowiazania->powiazaniaFound()) throw new Exception("Nie znaleziono żadnych powiązań");
 			file_put_contents($xmlFile, $BiAuditPowiazania->asXml());
 			$BiAuditPowiazania->saveToDb();
-			$BiAuditPowiazania->generatePdfAndHtml($tasksDirLocation);
+			$BiAuditPowiazania->generatePdfAndHtml();
 			file_put_contents($resultFile, "ok");
 		} catch (Exception $e) {
 			file_put_contents($resultFile, $e->getMessage());
@@ -1914,9 +1914,10 @@ class BiAuditRelations {
 class BiAuditPowiazania {
 
 	private $minDepth;
-	private $maxDepth;
+	private $maxDepth, $origMaxDepth;
 	private $onlyTargets;
 	private $ID;
+	private $nodes = [];
 	private $endNodes;
 	private $path = [];
 	private $results = [];
@@ -1925,7 +1926,12 @@ class BiAuditPowiazania {
 	private $destTables = ['BI_audit_ENERGA_RUM_KONTRAHENCI', 'BI_audit_KW_requested_person'];
 	private $refTables = [];
 	private $step = 0;
+	private $tasksDirLocation;
 	private $progressFile;
+	private $lastProgress = 0;
+	private $startTimestamp = null;
+	private $lowProgressCount = 0;
+	private $count, $i;
 
 	private function getRefTable($tableFrom, $tableTo, $returnException = true) {
 		if (!isset($this->refTables[$tableFrom][$tableTo])) {
@@ -1939,8 +1945,29 @@ class BiAuditPowiazania {
 		return $this->refTables[$tableFrom][$tableTo];
 	}
 
-	public function __construct($ID = 0) {
+	private function generateJson($type, $data) {
+		return json_encode(['ts' => microtime(true), 'type' => $type, 'item' => "{$this->i}/{$this->count}", 'data' => $data]);
+	}
+
+	private function saveToLog($message) {
+		$messageJson = $this->generateJson('message', ['message' => $message]);
+		echo $messageJson . "\n";
+	}
+	private function saveProgress($progress) {
+		$this->lastProgress = $progress;
+		$timestamp = microtime(true);
+		if ($progress) $estimated = ($timestamp - $this->startTimestamp) / $progress;
+		else $estimated = "N/A";
+		$progressJson = $this->generateJson('progress', ['progress' => $progress, 'results' => count($this->results), 'step' => $this->step, 'estimated' => $estimated]);
+		echo $progressJson . "\n";
+		file_put_contents($this->progressFile, $progressJson);
+	}
+
+	public function __construct($ID = 0, $tasksDirLocation = null) {
 		if (!$ID) throw new Exception("Wrong ID parameter");
+		if (!$tasksDirLocation) throw new Exception("Wrong directory of tasks location");
+		$this->tasksDirLocation = $tasksDirLocation;
+
 		$query = "select BI_analiza_minDepth, BI_analiza_maxDepth, BI_analiza_onlyTargets from BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA where ID = '{$ID}' and FILE_STATUS = 'IN_PROGRESS' and BI_analiza_reloadCache not in ('Full', 'Part')";
 		$result = DB::getPDO()->fetchAll($query);
 		if (!$result) throw new Exception("Błąd danych");
@@ -1962,43 +1989,68 @@ class BiAuditPowiazania {
 		if (!$result) throw new Exception("Błąd danych - nie zdefinowano żadnego końcowego obiektu");
 		$this->endNodes = array_map('reset', $result);
 
-		$this->progressFile = dirname(__FILE__) . "/generatePowiazania-{$ID}.progress";
+		$this->progressFile = "{$this->tasksDirLocation}/generatePowiazania-{$ID}.progress";
 
 		$refPowiazaniaToPracownicy = $this->getRefTable('BI_audit_ENERGA_RUM_KONTRAHENCI_POWIAZANIA', 'BI_audit_ENERGA_PRACOWNICY');
 		$query = "select `all`.ID as ID from BI_audit_ALL `all` join `{$refPowiazaniaToPracownicy}` ref
-			on `all`.REMOTE_TABLE = 'BI_audit_ENERGA_PRACOWNICY' and `all`.REMOTE_ID = ref.REMOTE_PRIMARY_KEY and ref.PRIMARY_KEY = '{$ID}'";
+			on `all`.REMOTE_TABLE = 'BI_audit_ENERGA_PRACOWNICY' and `all`.REMOTE_ID = ref.REMOTE_PRIMARY_KEY and ref.PRIMARY_KEY = '{$ID}' order by `all`.`ID` asc";
 		$result = DB::getPDO()->fetchAll($query);
-		$count = count($result);
-		foreach ($result as $i => $row) $this->findPowiazania($row['ID'], (($i + 1) / $count));
+		$this->count = count($result);
+		$this->saveProgress(0);
+		$this->origMaxDepth = $this->maxDepth;
+		foreach ($result as $this->i => $row) {
+			if ($this->maxDepth != $this->origMaxDepth) {
+				$this->maxDepth = $this->origMaxDepth;
+				$this->saveToLog("Przywrócono oryginalną maksymalną głębokość przeszukiwania ({$this->origMaxDepth})");
+			}
+			//$this->findPowiazania($row['ID'], ($i / $count), $count);
+			$this->findPowiazania($row['ID']);
+			
+			$this->saveProgress(1);
+		}
 	}
 
-	public function findPowiazania($ID, $progress, $relation = "", $end = 0) {
+	public function findPowiazania($ID, $progress = 0, $steps = 1, $relation = "", $end = 0) {
+		if (!$progress) {
+			$this->startTimestamp = microtime(true);
+			$this->lastProgress = 0;
+		}
+
+		if (((++$this->step) % 1000000) == 0) {
+			$progressDiff = $progress - $this->lastProgress;
+			$this->saveToLog("TEST: " . (number_format($progress - $this->lastProgress, 20)));
+			if ($progressDiff < 0.00005) {
+				$this->lowProgressCount++;
+				if ($this->lowProgressCount == 10) {
+					$this->maxDepth--;
+					$this->saveToLog("Zbyt dużo rekurencji, zmniejszono maksymalną głębokość poszukiwań do {$this->maxDepth} dla bieżącego pracownika");
+				}
+			} else $this->lowProgressCount = 0;
+			$this->saveProgress($progress);
+			if (memory_get_usage(true) > 4000000000) {
+				$this->nodes = [];
+				$this->saveToLog("Too much memory used, cleaning relations cache");
+			}
+		}
+
 		if (isset($this->path[$ID])) return false;
 
 		$this->path[$ID] = $relation;
 		if (!$relation) $relation = 0;
 
-		$finish = false;
-
 		if ($end) {
 			if (in_array($ID, $this->endNodes) && count($this->path) > $this->minDepth) $this->results[] = $this->path;
-			$finish = true;
+			array_pop($this->path);
+			return;
 		}
 
 		if (count($this->path) > $this->maxDepth) {
 			if (!$this->onlyTargets) $this->results[] = $this->path;
-			$finish = true;
-		}
-
-		if ($finish) {
 			array_pop($this->path);
-			if (((++$this->step) % 1000) == 0) {
-				$progressJson = json_encode(['progress' => $progress, 'results' => count($this->path), 'step' => $this->step]);
-				file_put_contents($this->progressFile, $progressJson);
-			}
 			return;
 		}
 
+/*
 //		$nodes = [];
 		if ($relation) $where = "and ({$relation} & ref.RELATION_ID) != {$relation}";
 		else $where = "";
@@ -2007,7 +2059,22 @@ class BiAuditPowiazania {
 //		foreach ($result as $row) $nodes[$row['ID2']] = ['relation' => (int) $row['RELATION_ID'], 'end' => $row['END']];
 //		foreach ($nodes as $node => $data) $this->findPowiazania($node, $data['relation'], $data['end']);
 		$count = count($result);
-		foreach ($result as $i => $row) $this->findPowiazania($row['ID2'], ($progress * ($i + 1) / $count), $row['RELATION_ID'], $row['END']);
+		foreach ($result as $i => $row) $this->findPowiazania($row['ID2'], ($progress + ($i / ($count * $steps))), ($count * $steps), $row['RELATION_ID'], $row['END']);
+*/
+
+		if (!isset($this->nodes[$ID][$relation])) {
+			if ($relation) $where = "and ({$relation} & ref.RELATION_ID) != {$relation}";
+			$query = "select ref.ID2, RELATION_ID, END from BI_audit_ALL_ref ref where ref.ID1 = {$ID} {$where}";
+			$this->nodes[$ID][$relation] = DB::getPDO()->fetchAll($query);
+		}
+		$nodes = array_values(array_filter($this->nodes[$ID][$relation], function ($node) {
+//			if ($relation && (($relation & $node['RELATION_ID']) == $relation)) return false;
+			if (isset($this->path[$node['ID2']])) return false;
+			return true;
+		}));
+		$count = count($nodes);// $i = 0;
+		foreach ($nodes as $i => $node) $this->findPowiazania($node['ID2'], ($progress + ($i / ($count * $steps))), ($count * $steps), $node['RELATION_ID'], $node['END']);
+
 		array_pop($this->path);
 	}
 
@@ -2125,11 +2192,11 @@ class BiAuditPowiazania {
 		else return false;
 	}
 
-	public function generatePdfAndHtml($tasksDirLocation) {
-		$xmlFile = "{$tasksDirLocation}/generatePowiazania-{$this->ID}.xml";
-		$pdfDestFile = "{$tasksDirLocation}/generatePowiazania-{$this->ID}.pdf";
-		$htmlDestDir = "{$tasksDirLocation}/generatePowiazania-{$this->ID}.html";
-		$htmlZipDestFile = "{$tasksDirLocation}/generatePowiazania-{$this->ID}.zip";
+	public function generatePdfAndHtml() {
+		$xmlFile = "{$this->tasksDirLocation}/generatePowiazania-{$this->ID}.xml";
+		$pdfDestFile = "{$this->tasksDirLocation}/generatePowiazania-{$this->ID}.pdf";
+		$htmlDestDir = "{$this->tasksDirLocation}/generatePowiazania-{$this->ID}.html";
+		$htmlZipDestFile = "{$this->tasksDirLocation}/generatePowiazania-{$this->ID}.zip";
 		$antDir = "/Library/Server/Web/Data/Sites/Default/SE/schema/WPS_Functions/default_db/recurse_ant/out";
 		$antXmlFilename = "relations-{$this->ID}.out_task.dita.final_with_RelatedFeatureLoop.xml";
 		$antXmlFile = $antDir . "/" . $antXmlFilename;