Optimizations: DBObjectSet::Count does it without loading all the data + possibility to specify LIMIT + restored the query cache (was inoperant) + improved the debug reports + added two settings (query_cache_enabled and debug_queries)

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@756 a333f486-631f-4898-b8df-5754b55c2be0

application/cmdbabstract.class.inc.php

@@ -438,7 +438,6 @@ abstract class cmdbAbstractObject extends CMDBObject
 			$aAttribs[$sAttCode] = array('label' => MetaModel::GetLabel($sClassName, $sAttCode), 'description' => MetaModel::GetDescription($sClassName, $sAttCode));
 		}
 		$aValues = array();
-		$oSet->Seek(0);
 		$bDisplayLimit = isset($aExtraParams['display_limit']) ? $aExtraParams['display_limit'] : true;
 		$iMaxObjects = -1;
 		if ($bDisplayLimit)
@@ -446,8 +445,10 @@ abstract class cmdbAbstractObject extends CMDBObject
 			if ($oSet->Count() > utils::GetConfig()->GetMaxDisplayLimit())
 			{
 				$iMaxObjects = utils::GetConfig()->GetMinDisplayLimit();
+				$oSet->SetLimit($iMaxObjects);
 			}
 		}
+		$oSet->Seek(0);
 		while (($oObj = $oSet->Fetch()) && ($iMaxObjects != 0))
 		{
 			$aRow = array();

core/config.class.inc.php

@@ -42,6 +42,10 @@ define ('DEFAULT_LOG_ISSUE', true);
 define ('DEFAULT_LOG_WEB_SERVICE', true);
 define ('DEFAULT_LOG_KPI_DURATION', false);
 define ('DEFAULT_LOG_KPI_MEMORY', false);
+define ('DEFAULT_DEBUG_QUERIES', false);
+
+define ('DEFAULT_QUERY_CACHE_ENABLED', true);
+
 
 define ('DEFAULT_MIN_DISPLAY_LIMIT', 10);
 define ('DEFAULT_MAX_DISPLAY_LIMIT', 15);
@@ -88,6 +92,8 @@ class Config
 	protected $m_bLogWebService;
 	protected $m_bLogKpiDuration; // private setting
 	protected $m_bLogKpiMemory; // private setting
+	protected $m_bDebugQueries; // private setting
+	protected $m_bQueryCacheEnabled; // private setting
 
 	/**
 	 * @var integer Number of elements to be displayed when there are more than m_iMaxDisplayLimit elements
@@ -300,6 +306,9 @@ class Config
 		$this->m_bLogWebService = isset($MySettings['log_web_service']) ? (bool) trim($MySettings['log_web_service']) : DEFAULT_LOG_WEB_SERVICE;
 		$this->m_bLogKPIDuration = isset($MySettings['log_kpi_duration']) ? (bool) trim($MySettings['log_kpi_duration']) : DEFAULT_LOG_KPI_DURATION;
 		$this->m_bLogKPIMemory = isset($MySettings['log_kpi_memory']) ? (bool) trim($MySettings['log_kpi_memory']) : DEFAULT_LOG_KPI_MEMORY;
+		$this->m_bDebugQueries = isset($MySettings['debug_queries']) ? (bool) trim($MySettings['debug_queries']) : DEFAULT_DEBUG_QUERIES;
+		$this->m_bQueryCacheEnabled = isset($MySettings['query_cache_enabled']) ? (bool) trim($MySettings['query_cache_enabled']) : DEFAULT_QUERY_CACHE_ENABLED;
+
 		$this->m_iMinDisplayLimit = isset($MySettings['min_display_limit']) ? trim($MySettings['min_display_limit']) : DEFAULT_MIN_DISPLAY_LIMIT;
 		$this->m_iMaxDisplayLimit = isset($MySettings['max_display_limit']) ? trim($MySettings['max_display_limit']) : DEFAULT_MAX_DISPLAY_LIMIT;
 		$this->m_iStandardReloadInterval = isset($MySettings['standard_reload_interval']) ? trim($MySettings['standard_reload_interval']) : DEFAULT_STANDARD_RELOAD_INTERVAL;
@@ -451,6 +460,16 @@ class Config
 		return $this->m_bLogKPIMemory;
 	}
 
+	public function GetDebugQueries()
+	{
+		return $this->m_bDebugQueries;
+	}
+
+	public function GetQueryCacheEnabled()
+	{
+		return $this->m_bQueryCacheEnabled;
+	}
+
 	public function GetMinDisplayLimit()
 	{
 		return $this->m_iMinDisplayLimit;

core/dbobjectset.class.php

@@ -38,11 +38,13 @@ class DBObjectSet
 	private $m_aId2Row;
 	private $m_iCurrRow;
 
-	public function __construct(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array())
+	public function __construct(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0)
 	{
 		$this->m_oFilter = $oFilter;
 		$this->m_aOrderBy = $aOrderBy;
 		$this->m_aArgs = $aArgs;
+		$this->m_iLimitCount = $iLimitCount;
+		$this->m_iLimitStart = $iLimitStart;
 
 		$this->m_bLoaded = false;
 		$this->m_aData = array(); // array of (row => array of (classalias) => object)
@@ -194,11 +196,33 @@ class DBObjectSet
 		return MetaModel::GetRootClass($this->GetClass());
 	}
 
+	public function SetLimit($iLimitCount, $iLimitStart = 0)
+	{
+		$this->m_iLimitCount = $iLimitCount;
+		$this->m_iLimitStart = $iLimitStart;
+	}
+
+	public function GetLimitCount()
+	{
+		return $this->m_iLimitCount;
+	}
+
+	public function GetLimitStart()
+	{
+		return $this->m_iLimitStart;
+	}
+
 	public function Load()
 	{
 		if ($this->m_bLoaded) return;
-
-		$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs);
+		if ($this->m_iLimitCount > 0)
+		{
+			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_iLimitCount, $this->m_iLimitStart);
+		}
+		else
+		{
+			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs);
+		}
 		$resQuery = CMDBSource::Query($sSQL);
 		if (!$resQuery) return;
 
@@ -220,8 +244,13 @@ class DBObjectSet
 
 	public function Count()
 	{
-		if (!$this->m_bLoaded) $this->Load();
-		return count($this->m_aData);
+		$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, 0, 0, true);
+		$resQuery = CMDBSource::Query($sSQL);
+		if (!$resQuery) return 0;
+
+		$aRow = CMDBSource::FetchArray($resQuery);
+		CMDBSource::FreeResult($resQuery);
+		return $aRow['COUNT'];
 	}
 
 	public function Fetch($sClassAlias = '')

core/kpi.class.inc.php

@@ -47,34 +47,50 @@ class ExecutionKPI
 	{
 		foreach (self::$m_aStats as $sOperation => $aOpStats)
 		{
-			echo "====================<br/>\n";
-			echo "KPIs for $sOperation<br/>\n";
-			echo "====================<br/>\n";
+			echo "<h2>KPIs for $sOperation</h2>\n";
 			$fTotalOp = 0;
 			$iTotalOp = 0;
 			$fMinOp = null;
 			$fMaxOp = 0;
+			echo "<ul>\n";
 			foreach ($aOpStats as $sArguments => $aEvents)
 			{
 				$fTotalInter = 0;
-				$iTotalInter = 0;
+				$fMinInter = null;
+				$fMaxInter = 0;
 				foreach ($aEvents as $fDuration)
 				{
 					$fTotalInter += $fDuration;
-					$iTotalInter++;
+					$fMinInter = is_null($fMinInter) ? $fDuration : min($fMinInter, $fDuration);
+					$fMaxInter = max($fMaxInter, $fDuration);
 
 					$fMinOp = is_null($fMinOp) ? $fDuration : min($fMinOp, $fDuration);
 					$fMaxOp = max($fMaxOp, $fDuration);
 				}
 				$fTotalOp += $fTotalInter;
 				$iTotalOp++;
-				echo "$sArguments: $iTotalInter (".round($fTotalInter, 3).")<br/>\n";
+
+				$iCountInter = count($aEvents);
+				$sTotalInter = round($fTotalInter, 3)."s";
+				if ($iCountInter > 1)
+				{
+					$sMinInter = round($fMinInter, 3)."s";
+					$sMaxInter = round($fMaxInter, 3)."s";
+					$sTimeDesc = "$sTotalInter (from $sMinInter to $sMaxInter) in $iCountInter times";
+				}
+				else
+				{
+					$sTimeDesc = "$sTotalInter";
+				}
+				echo "<li>Spent $sTimeDesc, on: <span style=\"font-size:60%\">$sArguments</span></li>\n";
 			}
-			echo "Total: $iTotalOp (".round($fTotalOp, 3).")<br/>\n";
-			echo "Min: ".round($fMinOp, 3)."<br/>\n";
-			echo "Max: ".round($fMaxOp, 3)."<br/>\n";
-			echo "Avg: ".round($fTotalOp / $iTotalOp, 3)."<br/>\n";
-			echo "====================<br/>\n";
+			echo "</ul>\n";
+			echo "<ul>Sumary for $sOperation\n";
+			echo "<li>Total: $iTotalOp (".round($fTotalOp, 3).")</li>\n";
+			echo "<li>Min: ".round($fMinOp, 3)."</li>\n";
+			echo "<li>Max: ".round($fMaxOp, 3)."</li>\n";
+			echo "<li>Avg: ".round($fTotalOp / $iTotalOp, 3)."</li>\n";
+			echo "</ul>\n";
 		}
 	}
 

core/metamodel.class.php

@@ -202,7 +202,8 @@ abstract class MetaModel
 
 	private static $m_oConfig = null;
 
-	private static $m_bTraceQueries = true;
+	private static $m_bQueryCacheEnabled = false;
+	private static $m_bTraceQueries = false;
 	private static $m_aQueriesLog = array();
 	
 	private static $m_bLogIssue = false;
@@ -1539,7 +1540,7 @@ abstract class MetaModel
 		return $aScalarArgs;
 	}
 
-	public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array())
+	public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
 	{
 		// Hide objects that are not visible to the current user
 		//
@@ -1562,12 +1563,23 @@ abstract class MetaModel
 			}
 		}
 
+		if (self::$m_bQueryCacheEnabled || self::$m_bTraceQueries)
+		{
+			// Need to identify the query
+			$sOqlQuery = $oFilter->ToOql();
+			$sOqlId = md5($sOqlQuery);
+		}
+		else
+		{
+			$sOqlQuery = "SELECTING... ".$oFilter->GetClass();
+			$sOqlId = "query id ? n/a";
+		}
+
+
 		// Query caching
 		//
-		$bQueryCacheEnabled = true;
-		if ($bQueryCacheEnabled)
+		if (self::$m_bQueryCacheEnabled)
 		{
-			$sOqlQuery = $oFilter->ToOql();
 			// Warning: using directly the query string as the key to the hash array can FAIL if the string
 			// is long and the differences are only near the end... so it's safer (but not bullet proof?)
 			// to use a hash (like md5) of the string as the key !
@@ -1578,16 +1590,12 @@ abstract class MetaModel
 			// SELECT SLT JOIN lnkSLTToSLA AS L1 ON L1.slt_id=SLT.id JOIN SLA ON L1.sla_id = SLA.id JOIN lnkContractToSLA AS L2 ON L2.sla_id = SLA.id JOIN CustomerContract ON L2.contract_id = CustomerContract.id WHERE SLT.ticket_priority = 1 AND SLA.service_id = 3 AND SLT.metric = 'TTR' AND CustomerContract.customer_id = 2
 			// the only difference is R instead or O at position 285 (TTR instead of TTO)...
 			//
-			if (array_key_exists(md5($sOqlQuery), self::$m_aQueryStructCache))
+			if (array_key_exists($sOqlId, self::$m_aQueryStructCache))
 			{
 				// hit!
-				$oSelect = clone self::$m_aQueryStructCache[$sOqlQuery];
+				$oSelect = clone self::$m_aQueryStructCache[$sOqlId];
 			}
 		}
-		else
-		{
-			$sOqlQuery = "dummy";
-		}
 
 		if (!isset($oSelect))
 		{
@@ -1600,7 +1608,7 @@ abstract class MetaModel
 			$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oConditionTree, $aClassAliases, $aTableAliases, $aTranslation, $oFilter);
 			$oKPI->ComputeStats('MakeQuery (select)', $sOqlQuery);
 
-			self::$m_aQueryStructCache[$sOqlQuery] = clone $oSelect;
+			self::$m_aQueryStructCache[$sOqlId] = clone $oSelect;
 		}
 
 		// Check the order by specification, and prefix with the class alias
@@ -1636,7 +1644,7 @@ abstract class MetaModel
 
 		try
 		{
-			$sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs);
+			$sRes = $oSelect->RenderSelect($aOrderSpec, $aScalarArgs, $iLimitCount, $iLimitStart, $bGetCount);
 		}
 		catch (MissingQueryArgument $e)
 		{
@@ -1647,16 +1655,25 @@ abstract class MetaModel
 
 		if (self::$m_bTraceQueries)
 		{
-			$aParams = array();
-			if (!array_key_exists($sOqlQuery, self::$m_aQueriesLog))
+			$sQueryId = md5($sRes);
+			if(!isset(self::$m_aQueriesLog[$sOqlId]))
+			{
+				self::$m_aQueriesLog[$sOqlId]['oql'] = $sOqlQuery;
+				self::$m_aQueriesLog[$sOqlId]['hits'] = 1;
+			}
+			else
+			{
+				self::$m_aQueriesLog[$sOqlId]['hits']++;
+			}
+			if(!isset(self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]))
 			{
-				self::$m_aQueriesLog[$sOqlQuery] = array(
-					'sql' => array(),
-					'count' => 0,
-				);
+				self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['sql'] = $sRes;
+				self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count'] = 1;
+			}
+			else
+			{
+				self::$m_aQueriesLog[$sOqlId]['queries'][$sQueryId]['count']++;
 			}
-			self::$m_aQueriesLog[$sOqlQuery]['count']++;
-			self::$m_aQueriesLog[$sOqlQuery]['sql'][] = $sRes;
 		}
 
 		return $sRes;
@@ -1664,17 +1681,37 @@ abstract class MetaModel
 
 	public static function ShowQueryTrace()
 	{
-		$iTotal = 0;
-		foreach (self::$m_aQueriesLog as $sOql => $aQueryData)
+		if (!self::$m_bTraceQueries) return;
+
+		$iOqlCount = count(self::$m_aQueriesLog);
+		if ($iOqlCount == 0)
+		{
+			echo "<p>Trace activated, but no query found</p>\n";
+			return;
+		}
+
+		$iSqlCount = 0;
+		foreach (self::$m_aQueriesLog as $aOqlData)
 		{
-			echo "<h2>$sOql</h2>\n";
-			$iTotal += $aQueryData['count'];
-			echo '<p>'.$aQueryData['count'].'</p>';
-			echo '<p>Example: '.$aQueryData['sql'][0].'</p>';
+			$iSqlCount += $aOqlData['hits'];
+		}
+		echo "<h2>Stats on SELECT queries: OQL=$iOqlCount, SQL=$iSqlCount</h2>\n";
+
+		foreach (self::$m_aQueriesLog as $aOqlData)
+		{
+			$sOql = $aOqlData['oql'];
+			$sHits = $aOqlData['hits'];
+
+			echo "<p><b>$sHits</b> hits for OQL query: $sOql</p>\n";
+			echo "<ul id=\"ClassesRelationships\" class=\"treeview\">\n";
+			foreach($aOqlData['queries'] as $aSqlData)
+			{
+				$sQuery = $aSqlData['sql'];
+				$sSqlHits = $aSqlData['count'];
+				echo "<li><b>$sSqlHits</b> hits for SQL: <span style=\"font-size:60%\">$sQuery</span><li>\n";
+			}
+			echo "</ul>\n";
 		}
-		echo "<h2>Total</h2>\n";
-		echo "<p>Count of executed queries: $iTotal</p>";
-		echo "<p>Count of built queries: ".count(self::$m_aQueriesLog)."</p>";
 	}
 
 	public static function MakeDeleteQuery(DBObjectSearch $oFilter, $aArgs = array())
@@ -3211,6 +3248,9 @@ abstract class MetaModel
 			ExecutionKPI::EnableMemory();
 		}
 
+		self::$m_bTraceQueries = self::$m_oConfig->GetDebugQueries();
+		self::$m_bQueryCacheEnabled = self::$m_oConfig->GetQueryCacheEnabled();
+
 		// Note: load the dictionary as soon as possible, because it might be
 		//       needed when some error occur
 		foreach (self::$m_oConfig->GetDictionaries() as $sModule => $sToInclude)

core/sqlquery.class.inc.php

@@ -238,7 +238,7 @@ class SQLQuery
 	}
 
 	// Interface, build the SQL query
-	public function RenderSelect($aOrderBy = array(), $aArgs = array())
+	public function RenderSelect($aOrderBy = array(), $aArgs = array(), $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
 	{
 		// The goal will be to complete the lists as we build the Joins
 		$aFrom = array();
@@ -248,15 +248,31 @@ class SQLQuery
 		$aSetValues = array();
 		$this->privRender($aFrom, $aFields, $oCondition, $aDelTables, $aSetValues);
 
-		$sSelect = self::ClauseSelect($aFields);
 		$sFrom   = self::ClauseFrom($aFrom);
 		$sWhere  = self::ClauseWhere($oCondition, $aArgs);
-		$sOrderBy = self::ClauseOrderBy($aOrderBy);
-		if (!empty($sOrderBy))
+		if ($bGetCount)
 		{
-			$sOrderBy = "ORDER BY $sOrderBy";
+			$sSQL = "SELECT COUNT(*) AS COUNT FROM $sFrom WHERE $sWhere";
 		}
-		return "SELECT DISTINCT $sSelect FROM $sFrom WHERE $sWhere $sOrderBy";
+		else
+		{
+			$sSelect = self::ClauseSelect($aFields);
+			$sOrderBy = self::ClauseOrderBy($aOrderBy);
+			if (!empty($sOrderBy))
+			{
+				$sOrderBy = "ORDER BY $sOrderBy";
+			}
+			if ($iLimitCount > 0)
+			{
+				$sLimit = 'LIMIT '.$iLimitStart.', '.$iLimitCount;
+			}
+			else
+			{
+				$sLimit = '';
+			}
+			$sSQL = "SELECT DISTINCT $sSelect FROM $sFrom WHERE $sWhere $sOrderBy $sLimit";
+		}
+		return $sSQL;
 	}
 
 	private static function ClauseSelect($aFields)