N.1065 Fix performance issues.

Add statistics on query table join optimization.

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@4992 a333f486-631f-4898-b8df-5754b55c2be0

core/apc-emulation.php

@@ -21,6 +21,42 @@
  * Date: 27/09/2017
  */
 
+/**
+ * @param string $cache_type
+ * @param bool $limited
+ * @return array|bool
+ */
+function apc_cache_info($cache_type = '', $limited = false)
+{
+	$aInfo = array();
+	$sRootCacheDir = apc_emul_get_cache_filename('');
+	$aInfo['cache_list'] = apc_emul_get_cache_entries($sRootCacheDir);
+	return $aInfo;
+}
+
+function apc_emul_get_cache_entries($sEntry)
+{
+	$aResult = array();
+	if (is_dir($sEntry))
+	{
+		$aFiles = array_diff(scandir($sEntry), array('.', '..'));
+		foreach($aFiles as $sFile)
+		{
+			$sSubFile = $sEntry.'/'.$sFile;
+			$aResult = array_merge($aResult, apc_emul_get_cache_entries($sSubFile));
+		}
+	}
+	else
+	{
+		$sKey = basename($sEntry);
+		if (strpos($sKey, '-') === 0)
+		{
+			$sKey = substr($sKey, 1);
+		}
+		$aResult[] = array('info' => $sKey);
+	}
+	return $aResult;
+}
 
 /**
  * @param array|string $key
@@ -188,6 +224,10 @@ function apc_emul_manage_new_entry($sNewFilename)
 	static $aFilesByTime = null;
 	static $iFileCount = 0;
 	$iMaxFiles = MetaModel::GetConfig()->Get('apc_cache_emulation.max_entries');
+	if ($iMaxFiles == 0)
+	{
+		return;
+	}
 	if (!$aFilesByTime)
 	{
 		$sRootCacheDir = apc_emul_get_cache_filename('');
@@ -203,7 +243,7 @@ function apc_emul_manage_new_entry($sNewFilename)
 		$aFilesByTime[$sNewFilename] = time();
 		$iFileCount++;
 	}
-	if (($iMaxFiles !== 0) && ($iFileCount > $iMaxFiles))
+	if ($iFileCount > $iMaxFiles)
 	{
 		$iFileNbToRemove = $iFileCount - $iMaxFiles;
 		foreach($aFilesByTime as $sFileToRemove => $iTime)

core/dbobjectsearch.class.php

@@ -1466,10 +1466,14 @@ class DBObjectSearch extends DBSearch
 
 		// Create a unique cache id
 		//
+		$aContextData = array();
 		if (self::$m_bQueryCacheEnabled || self::$m_bTraceQueries)
 		{
+			$aContextData['sRequestUri'] = $_SERVER['REQUEST_URI'];
+
 			// Need to identify the query
 			$sOqlQuery = $oSearch->ToOql(false, null, true);
+			$aContextData['sOqlQuery'] = $sOqlQuery;
 
 			if (count($aModifierProperties))
 			{
@@ -1480,12 +1484,14 @@ class DBObjectSearch extends DBSearch
 			{
 				$sModifierProperties = '';
 			}
+			$aContextData['aModifierProperties'] = $aModifierProperties;
 
 			$sRawId = $sOqlQuery.$sModifierProperties;
 			if (!is_null($aAttToLoad))
 			{
 				$sRawId .= json_encode($aAttToLoad);
 			}
+			$aContextData['aAttToLoad'] = $aAttToLoad;
 			if (!is_null($aGroupByExpr))
 			{
 				foreach($aGroupByExpr as $sAlias => $oExpr)
@@ -1493,13 +1499,19 @@ class DBObjectSearch extends DBSearch
 					$sRawId .= 'g:'.$sAlias.'!'.$oExpr->Render();
 				}
 			}
+			$aContextData['aGroupByExpr'] = $aGroupByExpr;
 			$sRawId .= $bGetCount;
 			if (is_array($aSelectedClasses))
 			{
 				$sRawId .= implode(',', $aSelectedClasses); // Unions may alter the list of selected columns
 			}
-			$sRawId .= $oSearch->GetArchiveMode() ? '--arch' : '';
-			$sRawId .= $oSearch->GetShowObsoleteData() ? '--obso' : '';
+			$aContextData['aSelectedClasses'] = $aSelectedClasses;
+			$bIsArchiveMode = $oSearch->GetArchiveMode();
+			$sRawId .= $bIsArchiveMode ? '--arch' : '';
+			$bShowObsoleteData = $oSearch->GetShowObsoleteData();
+			$sRawId .= $bShowObsoleteData ? '--obso' : '';
+			$aContextData['bIsArchiveMode'] = $bIsArchiveMode;
+			$aContextData['bShowObsoleteData'] = $bShowObsoleteData;
 			$sOqlId = md5($sRawId);
 		}
 		else
@@ -1557,6 +1569,7 @@ class DBObjectSearch extends DBSearch
 			{
 				if (self::$m_bUseAPCCache)
 				{
+					$oSQLQuery->m_aContextData = $aContextData;
 					$oKPI = new ExecutionKPI();
 					apc_store($sOqlAPCCacheId, $oSQLQuery, self::$m_iQueryCacheTTL);
 					$oKPI->ComputeStats('Query APC (store)', $sOqlQuery);
@@ -1568,6 +1581,14 @@ class DBObjectSearch extends DBSearch
 		return $oSQLQuery;
 	}
 
+	/**
+	 * @param $aAttToLoad
+	 * @param $bGetCount
+	 * @param $aModifierProperties
+	 * @param null $aGroupByExpr
+	 * @param null $aSelectedClasses
+	 * @return null|SQLObjectQuery
+	 */
 	protected function BuildSQLQueryStruct($aAttToLoad, $bGetCount, $aModifierProperties, $aGroupByExpr = null, $aSelectedClasses = null)
 	{
 		$oBuild = new QueryBuilderContext($this, $aModifierProperties, $aGroupByExpr, $aSelectedClasses);
@@ -1612,6 +1633,12 @@ class DBObjectSearch extends DBSearch
 	}
 
 
+	/**
+	 * @param $oBuild
+	 * @param null $aAttToLoad
+	 * @param array $aValues
+	 * @return null|SQLObjectQuery
+	 */
 	protected function MakeSQLObjectQuery(&$oBuild, $aAttToLoad = null, $aValues = array())
 	{
 		// Note: query class might be different than the class of the filter

core/sqlobjectquery.class.inc.php

@@ -36,7 +36,8 @@
 
 class SQLObjectQuery extends SQLQuery
 {
-	private $m_SourceOQL = '';
+	public $m_aContextData = null;
+	public $m_iOriginalTableCount = 0;
 	private $m_sTable = '';
 	private $m_sTableAlias = '';
 	private $m_aFields = array();
@@ -510,6 +511,7 @@ class SQLObjectQuery extends SQLQuery
 
 	public function OptimizeJoins($aUsedTables, $bTopCall = true)
 	{
+		$this->m_iOriginalTableCount = $this->CountTables();
 		if ($bTopCall)
 		{
 			// Top call: complete the list of tables absolutely required to perform the right query
@@ -612,4 +614,5 @@ class SQLObjectQuery extends SQLQuery
 		// None of the tables is in the list of required tables
 		return $bResult;
 	}
+
 }

test/display_cache_content.php

@@ -0,0 +1,103 @@
+<?php
+// Copyright (c) 2010-2017 Combodo SARL
+//
+//   This file is part of iTop.
+//
+//   iTop is free software; you can redistribute it and/or modify
+//   it under the terms of the GNU Affero General Public License as published by
+//   the Free Software Foundation, either version 3 of the License, or
+//   (at your option) any later version.
+//
+//   iTop is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU Affero General Public License for more details.
+//
+//   You should have received a copy of the GNU Affero General Public License
+//   along with iTop. If not, see <http://www.gnu.org/licenses/>
+//
+
+/**
+ * Date: 06/10/2017
+ */
+
+require_once('../approot.inc.php');
+require_once(APPROOT.'application/startup.inc.php');
+
+
+$sEnvironment = MetaModel::GetEnvironmentId();
+$aEntries = array();
+$aCacheUserData = apc_cache_info_compat();
+if (is_array($aCacheUserData) && isset($aCacheUserData['cache_list']))
+{
+	$sPrefix = 'itop-'.$sEnvironment.'-query-cache-';
+
+	foreach($aCacheUserData['cache_list'] as $i => $aEntry)
+	{
+		$sEntryKey = array_key_exists('info', $aEntry) ? $aEntry['info'] : $aEntry['key'];
+		if (strpos($sEntryKey, $sPrefix) === 0)
+		{
+			$aEntries[] = $sEntryKey;
+		}
+	}
+}
+
+echo "<pre>";
+
+if (empty($aEntries))
+{
+	return;
+}
+
+$sKey = $aEntries[0];
+$result = apc_fetch($sKey);
+if (!is_object($result))
+{
+	return;
+}
+$oSQLQuery = $result;
+
+echo "NB Tables before;NB Tables after;";
+foreach($oSQLQuery->m_aContextData as $sField => $oValue)
+{
+	echo $sField.';';
+}
+echo "\n";
+
+sort($aEntries);
+
+foreach($aEntries as $sKey)
+{
+	$result = apc_fetch($sKey);
+	if (is_object($result))
+	{
+		$oSQLQuery = $result;
+		if (isset($oSQLQuery->m_aContextData))
+		{
+			echo $oSQLQuery->m_iOriginalTableCount.";".$oSQLQuery->CountTables().';';
+			foreach($oSQLQuery->m_aContextData as $oValue)
+			{
+				if (is_array($oValue))
+				{
+					$sVal = json_encode($oValue);
+				}
+				else
+				{
+					if (empty($oValue))
+					{
+						$sVal = '';
+					}
+					else
+					{
+						$sVal = $oValue;
+					}
+				}
+				echo '"'.$sVal.'"'.';';
+			}
+			echo "\n";
+		}
+	}
+}
+
+echo "</pre>";
+