#122 Optimized the load of data set (do not load unused columns, was provoking the swapping of MySQL tmp memory tables)

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@1281 a333f486-631f-4898-b8df-5754b55c2be0

application/cmdbabstract.class.inc.php

@@ -562,6 +562,7 @@ abstract class cmdbAbstractObject extends CMDBObject implements iDisplay
 		$sZListName = isset($aExtraParams['zlist']) ? ($aExtraParams['zlist']) : 'list';
 		$aList = self::FlattenZList(MetaModel::GetZListItems($sClassName, $sZListName));
 		$aList = array_merge($aList, $aExtraFields);
+
 		// Filter the list to removed linked set since we are not able to display them here
 		foreach($aList as $index => $sAttCode)
 		{
@@ -572,6 +573,11 @@ abstract class cmdbAbstractObject extends CMDBObject implements iDisplay
 				unset($aList[$index]);
 			}
 		}
+
+		// Load only the requested columns
+		$sClassAlias = $oSet->GetFilter()->GetClassAlias();
+		$oSet->OptimizeColumnLoad(array($sClassAlias => $aList));
+
 		if (!empty($sLinkageAttribute))
 		{
 			// The set to display is in fact a set of links between the object specified in the $sLinkageAttribute
@@ -802,6 +808,17 @@ EOF
 				$aAttribs[$sAttCode.'_'.$sAlias] = array('label' => MetaModel::GetLabel($sClassName, $sAttCode), 'description' => MetaModel::GetDescription($sClassName, $sAttCode));
 			}
 		}
+		// Load only the requested columns
+		$aAttToLoad = array(); // attributes to load
+		foreach($aAuthorizedClasses as $sAlias => $sClassName)
+		{
+			foreach($aList[$sClassName] as $sAttCode)
+			{
+				$aAttToLoad[$sAlias][] = $sAttCode;
+			}
+		}
+		$oSet->OptimizeColumnLoad($aAttToLoad);
+
 		$aValues = array();
 		$oSet->Seek(0);
 		$bDisplayLimit = isset($aExtraParams['display_limit']) ? $aExtraParams['display_limit'] : true;

core/dbobject.class.php

@@ -62,15 +62,15 @@ abstract class DBObject
 	protected $m_oMasterReplicaSet = null; // Set of SynchroReplica related to this object
 
 	// Use the MetaModel::NewObject to build an object (do we have to force it?)
-	public function __construct($aRow = null, $sClassAlias = '', $aExtendedDataSpec = null)
+	public function __construct($aRow = null, $sClassAlias = '', $aAttToLoad = null, $aExtendedDataSpec = null)
 	{
 		if (!empty($aRow))
 		{
-			$this->FromRow($aRow, $sClassAlias, $aExtendedDataSpec);
+			$this->FromRow($aRow, $sClassAlias, $aAttToLoad, $aExtendedDataSpec);
 			$this->m_bFullyLoaded = $this->IsFullyLoaded();
 			return;
 		}
-		// Creation of brand new object
+		// Creation of a brand new object
 		//
 
 		$this->m_iKey = self::GetNextTempId(get_class($this));
@@ -82,8 +82,8 @@ abstract class DBObject
 			$this->m_aOrigValues[$sAttCode] = null;
 			if ($oAttDef->IsExternalField())
 			{
-				// This field has to be read from the DB 
-				$this->m_aLoadedAtt[$sAttCode] = false;
+				// This field has to be read from the DB
+				// Leave the flag unset (optimization) 
 			}
 			else
 			{
@@ -194,7 +194,7 @@ abstract class DBObject
 		$this->m_bFullyLoaded = true;
 	}
 
-	protected function FromRow($aRow, $sClassAlias = '', $aExtendedDataSpec = null)
+	protected function FromRow($aRow, $sClassAlias = '', $aAttToLoad = null, $aExtendedDataSpec = null)
 	{
 		if (strlen($sClassAlias) == 0)
 		{
@@ -235,11 +235,17 @@ abstract class DBObject
 		// Build the object from an array of "attCode"=>"value")
 		//
 		$bFullyLoaded = true; // ... set to false if any attribute is not found
-		foreach(MetaModel::ListAttributeDefs(get_class($this)) as $sAttCode=>$oAttDef)
+		if (is_null($aAttToLoad) || !array_key_exists($sClassAlias, $aAttToLoad))
+		{
+			$aAttList = MetaModel::ListAttributeDefs(get_class($this));
+		}
+		else
+		{
+			$aAttList = $aAttToLoad[$sClassAlias];
+		}
+		
+		foreach($aAttList as $sAttCode=>$oAttDef)
 		{
-			// Say something, whatever the type of attribute
-			$this->m_aLoadedAtt[$sAttCode] = false;
-
 			// Skip links (could not be loaded by the mean of this query)
 			if ($oAttDef->IsLinkSet()) continue;
 
@@ -366,7 +372,7 @@ abstract class DBObject
 		{
 			throw new CoreException("Unknown attribute code '$sAttCode' for the class ".get_class($this));
 		}
-		if ($this->m_bIsInDB && !$this->m_aLoadedAtt[$sAttCode] && !$this->m_bDirty)
+		if ($this->m_bIsInDB && !isset($this->m_aLoadedAtt[$sAttCode]) && !$this->m_bDirty)
 		{
 			// #@# non-scalar attributes.... handle that differently
 			$this->Reload();
@@ -616,23 +622,7 @@ abstract class DBObject
 
 	public function GetName()
 	{
-		$aNameSpec = MetaModel::GetNameSpec(get_class($this));
-		$sFormat = $aNameSpec[0];
-		$aAttributes = $aNameSpec[1];                                                     
-
-		$aValues = array();
-      foreach ($aAttributes as $sAttCode)
-      {
-      	if (empty($sAttCode))
-      	{
-      		$aValues[] = $this->m_iKey;
-			}
-			else
-			{
-				$aValues[] = $this->Get($sAttCode);
-			}
-		}
-		return vsprintf($sFormat, $aValues);
+		return $this->Get('friendlyname');
 	}
 
 	public function GetState()

core/dbobjectset.class.php

@@ -43,6 +43,7 @@ class DBObjectSet
 		$this->m_oFilter = $oFilter;
 		$this->m_aOrderBy = $aOrderBy;
 		$this->m_aArgs = $aArgs;
+		$this->m_aAttToLoad = null;
 		$this->m_aExtendedDataSpec = $aExtendedDataSpec;
 		$this->m_iLimitCount = $iLimitCount;
 		$this->m_iLimitStart = $iLimitStart;
@@ -78,6 +79,39 @@ class DBObjectSet
 		return $sRet;
 	}
 
+	public function OptimizeColumnLoad($aAttToLoad)
+	{
+		if (is_null($aAttToLoad))
+		{
+			$this->m_aAttToLoad = null;
+		}
+		else
+		{
+			// Complete the attribute list with the attribute codes
+			$aAttToLoadWithAttDef = array();
+			foreach($aAttToLoad as $sClassAlias => $aAttList)
+			{
+				$aSelectedClasses = $this->m_oFilter->GetSelectedClasses();
+				$sClass = $aSelectedClasses[$sClassAlias];
+				foreach($aAttList as $sAttToLoad)
+				{
+					$oAttDef = MetaModel::GetAttributeDef($sClass, $sAttToLoad);
+					$aAttToLoadWithAttDef[$sClassAlias][$sAttToLoad] = $oAttDef;
+					if ($oAttDef->IsExternalKey())
+					{
+						// Add the external key friendly name anytime
+						$oFriendlyNameAttDef = MetaModel::GetAttributeDef($sClass, $sAttToLoad.'_friendlyname');
+						$aAttToLoadWithAttDef[$sClassAlias][$sAttToLoad.'_friendlyname'] = $oFriendlyNameAttDef;
+					}
+				}
+				// Add the friendly name anytime
+				$oFriendlyNameAttDef = MetaModel::GetAttributeDef($sClass, 'friendlyname');
+				$aAttToLoadWithAttDef[$sClassAlias]['friendlyname'] = $oFriendlyNameAttDef;
+			}
+			$this->m_aAttToLoad = $aAttToLoadWithAttDef;
+		}
+	}
+
 	static public function FromObject($oObject)
 	{
 		$oRetSet = self::FromScratch(get_class($oObject));
@@ -270,11 +304,11 @@ class DBObjectSet
 
 		if ($this->m_iLimitCount > 0)
 		{
-			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_aExtendedDataSpec, $this->m_iLimitCount, $this->m_iLimitStart);
+			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_aAttToLoad, $this->m_aExtendedDataSpec, $this->m_iLimitCount, $this->m_iLimitStart);
 		}
 		else
 		{
-			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_aExtendedDataSpec);
+			$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, $this->m_aAttToLoad, $this->m_aExtendedDataSpec);
 		}
 		$resQuery = CMDBSource::Query($sSQL);
 		if (!$resQuery) return;
@@ -291,7 +325,7 @@ class DBObjectSet
 				}
 				else
 				{
-					$oObject = MetaModel::GetObjectByRow($sClass, $aRow, $sClassAlias, $this->m_aExtendedDataSpec);
+					$oObject = MetaModel::GetObjectByRow($sClass, $aRow, $sClassAlias, $this->m_aAttToLoad, $this->m_aExtendedDataSpec);
 				}
 				$aObjects[$sClassAlias] = $oObject;
 			}
@@ -310,7 +344,7 @@ class DBObjectSet
 		{
 			if (is_null($this->m_iCount))
 			{
-				$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, null, 0, 0, true);
+				$sSQL = MetaModel::MakeSelectQuery($this->m_oFilter, $this->m_aOrderBy, $this->m_aArgs, null, null, 0, 0, true);
 				$resQuery = CMDBSource::Query($sSQL);
 				if (!$resQuery) return 0;
 		

core/metamodel.class.php

@@ -1778,7 +1778,7 @@ abstract class MetaModel
 		return $aScalarArgs;
 	}
 
-	public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $aExtendedDataSpec = null, $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
+	public static function MakeSelectQuery(DBObjectSearch $oFilter, $aOrderBy = array(), $aArgs = array(), $aAttToLoad = null, $aExtendedDataSpec = null, $iLimitCount = 0, $iLimitStart = 0, $bGetCount = false)
 	{
 		// Hide objects that are not visible to the current user
 		//
@@ -1805,7 +1805,16 @@ abstract class MetaModel
 		{
 			// Need to identify the query
 			$sOqlQuery = $oFilter->ToOql();
-			$sOqlId = md5($sOqlQuery);
+
+			$sRawId = $sOqlQuery;
+			if (!is_null($aAttToLoad))
+			{
+				foreach($aAttToLoad as $sAlias => $aAttributes)
+				{
+					$sRawId = $sOqlQuery.'|'.implode(',', array_keys($aAttributes));
+				}
+			}
+			$sOqlId = md5($sRawId);
 		}
 		else
 		{
@@ -1851,6 +1860,43 @@ abstract class MetaModel
 			}
 		}
 
+		// Check the order by specification, and prefix with the class alias
+		// and make sure that the ordering columns are going to be selected
+		//
+		$aOrderSpec = array();
+		foreach ($aOrderBy as $sFieldAlias => $bAscending)
+		{
+			MyHelpers::CheckValueInArray('field name in ORDER BY spec', $sFieldAlias, self::GetAttributesList($oFilter->GetFirstJoinedClass()));
+			if (!is_bool($bAscending))
+			{
+				throw new CoreException("Wrong direction in ORDER BY spec, found '$bAscending' and expecting a boolean value");
+			}
+			$sFirstClassAlias = $oFilter->GetFirstJoinedClassAlias();
+			$aOrderSpec[$sFirstClassAlias.$sFieldAlias] = $bAscending;
+
+			// Make sure that the columns used for sorting are present in the loaded columns
+			if (!is_null($aAttToLoad) && !isset($aAttToLoad[$sFirstClassAlias][$sFieldAlias]))
+			{
+				$aAttToLoad[$sFirstClassAlias][$sFieldAlias] = MetaModel::GetAttributeDef($oFilter->GetFirstJoinedClass(), $sFieldAlias);
+			}			
+		}
+		// By default, force the name attribute to be the ordering key
+		//
+		if (empty($aOrderSpec))
+		{
+			foreach ($oFilter->GetSelectedClasses() as $sSelectedAlias => $sSelectedClass)
+			{
+				// By default, simply order on the "friendlyname" attribute, ascending
+				$aOrderSpec[$sSelectedAlias."friendlyname"] = true;
+
+				// Make sure that the columns used for sorting are present in the loaded columns
+				if (!is_null($aAttToLoad) && !isset($aAttToLoad[$sSelectedAlias]["friendlyname"]))
+				{
+					$aAttToLoad[$sSelectedAlias]["friendlyname"] = MetaModel::GetAttributeDef($sSelectedClass, "friendlyname");
+				}			
+			}
+		}
+
 		if (!isset($oSelect))
 		{
 			$aClassAliases = array();
@@ -1858,7 +1904,7 @@ abstract class MetaModel
 			$oQBExpr = new QueryBuilderExpressions(array(), $oFilter->GetCriteria());
 
 			$oKPI = new ExecutionKPI();
-			$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, array(), true /* main query */);
+			$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, $aAttToLoad, array(), true /* main query */);
 			$oSelect->SetSourceOQL($sOqlQuery);
 			$oKPI->ComputeStats('MakeQuery (select)', $sOqlQuery);
 
@@ -1875,29 +1921,6 @@ abstract class MetaModel
 			}
 		}
 
-		// Check the order by specification, and prefix with the class alias
-		//
-		$aOrderSpec = array();
-		foreach ($aOrderBy as $sFieldAlias => $bAscending)
-		{
-			MyHelpers::CheckValueInArray('field name in ORDER BY spec', $sFieldAlias, self::GetAttributesList($oFilter->GetFirstJoinedClass()));
-			if (!is_bool($bAscending))
-			{
-				throw new CoreException("Wrong direction in ORDER BY spec, found '$bAscending' and expecting a boolean value");
-			}
-			$aOrderSpec[$oFilter->GetFirstJoinedClassAlias().$sFieldAlias] = $bAscending;
-		}
-		// By default, force the name attribute to be the ordering key
-		//
-		if (empty($aOrderSpec))
-		{
-			foreach ($oFilter->GetSelectedClasses() as $sSelectedAlias => $sSelectedClass)
-			{
-				// By default, simply order on the "friendlyname" attribute, ascending
-				$aOrderSpec[$sSelectedAlias."friendlyname"] = true;
-			}
-		}
-
 		// Join to an additional table, if required...
 		//
 		if ($aExtendedDataSpec != null)
@@ -1994,7 +2017,7 @@ abstract class MetaModel
 		$aClassAliases = array();
 		$aTableAliases = array();
 		$oQBExpr = new QueryBuilderExpressions(array(), $oFilter->GetCriteria());
-		$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, array(), true /* main query */);
+		$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, null, array(), true /* main query */);
 		$aScalarArgs = array_merge(self::PrepareQueryArguments($aArgs), $oFilter->GetInternalParams());
 		return $oSelect->RenderDelete($aScalarArgs);
 	}
@@ -2005,12 +2028,12 @@ abstract class MetaModel
 		$aClassAliases = array();
 		$aTableAliases = array();
 		$oQBExpr = new QueryBuilderExpressions(array(), $oFilter->GetCriteria());
-		$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, $aValues, true /* main query */);
+		$oSelect = self::MakeQuery($oFilter->GetSelectedClasses(), $oQBExpr, $aClassAliases, $aTableAliases, $oFilter, null, $aValues, true /* main query */);
 		$aScalarArgs = array_merge(self::PrepareQueryArguments($aArgs), $oFilter->GetInternalParams());
 		return $oSelect->RenderUpdate($aScalarArgs);
 	}
 
-	private static function MakeQuery($aSelectedClasses, &$oQBExpr, &$aClassAliases, &$aTableAliases, DBObjectSearch $oFilter, $aValues = array(), $bIsMainQuery = false)
+	private static function MakeQuery($aSelectedClasses, &$oQBExpr, &$aClassAliases, &$aTableAliases, DBObjectSearch $oFilter, $aAttToLoad = null, $aValues = array(), $bIsMainQuery = false)
 	{
 		// Note: query class might be different than the class of the filter
 		// -> this occurs when we are linking our class to an external class (referenced by, or pointing to)
@@ -2033,7 +2056,15 @@ abstract class MetaModel
 			// default to the whole list of attributes + the very std id/finalclass
 			$oQBExpr->AddSelect($sClassAlias.'id', new FieldExpression('id', $sClassAlias));
 
-			foreach (self::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
+			if (is_null($aAttToLoad) || !array_key_exists($sClassAlias, $aAttToLoad))
+			{
+				$aAttList = self::ListAttributeDefs($sClass);
+			}
+			else
+			{
+				$aAttList = $aAttToLoad[$sClassAlias];
+			}
+			foreach ($aAttList as $sAttCode => $oAttDef)
 			{
 				if (!$oAttDef->IsScalar()) continue;
 				
@@ -2185,7 +2216,7 @@ abstract class MetaModel
 				$sForeignClassAlias = $oForeignFilter->GetFirstJoinedClassAlias();
 				$oQBExpr->PushJoinField(new FieldExpression($sForeignKeyAttCode, $sForeignClassAlias));
 
-				$oSelectForeign = self::MakeQuery($aSelectedClasses, $oQBExpr, $aClassAliases, $aTableAliases, $oForeignFilter);
+				$oSelectForeign = self::MakeQuery($aSelectedClasses, $oQBExpr, $aClassAliases, $aTableAliases, $oForeignFilter, $aAttToLoad);
 
 				$oJoinExpr = $oQBExpr->PopJoinField();
 				$sForeignKeyTable = $oJoinExpr->GetParent();
@@ -3888,7 +3919,7 @@ abstract class MetaModel
 		return $aRow;
 	}
 
-	public static function GetObjectByRow($sClass, $aRow, $sClassAlias = '', $aExtendedDataSpec = null)
+	public static function GetObjectByRow($sClass, $aRow, $sClassAlias = '', $aAttToLoad = null, $aExtendedDataSpec = null)
 	{
 		self::_check_subclass($sClass);	
 
@@ -3917,7 +3948,7 @@ abstract class MetaModel
 			// do the job for the real target class
 			$sClass = $aRow[$sClassAlias."finalclass"];
 		}
-		return new $sClass($aRow, $sClassAlias, $aExtendedDataSpec);
+		return new $sClass($aRow, $sClassAlias, $aAttToLoad, $aExtendedDataSpec);
 	}
 
 	public static function GetObject($sClass, $iKey, $bMustBeFound = true, $bAllowAllData = false)