#233 Speed up data load, found a bottleneck, but still some room for improvement, added internal settings to boost up huge load or workaround a bug in CheckToWrite (if any)

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@772 a333f486-631f-4898-b8df-5754b55c2be0

core/config.class.inc.php

@@ -75,6 +75,64 @@ class Config
 
 	protected $m_aModuleSettings;
 
+	// New way to store the settings !
+	//
+	protected $m_aSettings = array(
+		'skip_check_to_write' => array(
+			'type' => 'bool',
+			'description' => 'Disable data format and integrity checks to boost up data load (insert or update)',
+			'default' => false,
+			'value' => false,
+			'source_of_value' => '',
+			'show_in_conf_sample' => false,
+		),
+		'skip_check_ext_keys' => array(
+			'type' => 'bool',
+			'description' => 'Disable external key check when checking the value of attribtutes',
+			'default' => false,
+			'value' => false,
+			'source_of_value' => '',
+			'show_in_conf_sample' => false,
+		),
+	);
+
+	public function IsProperty($sPropCode)
+	{
+		return (array_key_exists($sPropCode, $this->m_aSettings));
+	}
+
+	public function Set($sPropCode, $value, $sSourceDesc = 'unknown')
+	{
+		$sType = $this->m_aSettings[$sPropCode]['type'];
+		switch($sType)
+		{
+		case 'bool':
+			$value = (bool) $value;
+			break;
+		case 'string':
+			$value = (string) $value;
+			break;
+		case 'integer':
+			$value = (integer) $value;
+			break;
+		case 'float':
+			$value = (float) $value;
+			break;
+		default:
+			throw new CoreException('Unknown type for setting', array('property' => $sPropCode, 'type' => $sType));
+		}
+
+		$this->m_aSettings[$sPropCode]['value'] = $value;
+		$this->m_aSettings[$sPropCode]['source_of_value'] = $sSourceDesc;
+
+	}
+
+	public function Get($sPropCode)
+	{
+		return $this->m_aSettings[$sPropCode]['value'];
+	}
+
+	// Those variables will be deprecated later, when the transition to ...Get('my_setting') will be done
 	protected $m_sDBHost;
 	protected $m_sDBUser;
 	protected $m_sDBPwd;
@@ -84,8 +142,9 @@ class Config
 	protected $m_sDBCollation;
 
 	/**
-	 * @var integer Event log options (see LOG_... definition)
+	 * Event log options (see LOG_... definition)
 	 */	 	
+	// Those variables will be deprecated later, when the transition to ...Get('my_setting') will be done
 	protected $m_bLogGlobal;
 	protected $m_bLogNotification;
 	protected $m_bLogIssue;
@@ -188,6 +247,11 @@ class Config
 			'../dictionaries/es_cr.dictionary.itop.core.php',	// Support for Spanish (from Costa Rica)
 		);
 
+		foreach($this->m_aSettings as $sPropCode => $aSettingInfo)
+		{
+			$this->m_aSettings[$sPropCode]['value'] = $aSettingInfo['default'];
+		}
+
 		$this->m_sDBHost = '';
 		$this->m_sDBUser = '';
 		$this->m_sDBPwd = '';
@@ -291,6 +355,15 @@ class Config
 		$this->m_aAddons = $MyModules['addons'];
 		$this->m_aDictionaries = $MyModules['dictionaries'];
 
+		foreach($MySettings as $sPropCode => $rawvalue)
+		{
+			if ($this->IsProperty($sPropCode))
+			{
+				$value = trim($rawvalue);
+				$this->Set($sPropCode, $value, $sConfigFile);
+			}
+		}
+
 		$this->m_sDBHost = trim($MySettings['db_host']);
 		$this->m_sDBUser = trim($MySettings['db_user']);
 		$this->m_sDBPwd = trim($MySettings['db_pwd']);
@@ -670,6 +743,22 @@ class Config
 			fwrite($hFile, "\n");
 			
 			fwrite($hFile, "\$MySettings = array(\n");
+			foreach($this->m_aSettings as $sPropCode => $aSettingInfo)
+			{
+				if ($aSettingInfo['show_in_conf_sample'])
+				{
+					$sType = $this->m_aSettings[$sPropCode]['type'];
+					switch($sType)
+					{
+					case 'bool':
+						$sSeenAs = $aSettingInfo['value'] ? '1' : '0';
+						break;
+					default:
+						$sSeenAs = "'".$aSettingInfo['value']."'";
+					}
+					fwrite($hFile, "\t'$sPropCode' => $sSeenAs,\n");
+				}
+			}
 			fwrite($hFile, "\t'db_host' => '{$this->m_sDBHost}',\n");
 			fwrite($hFile, "\t'db_user' => '{$this->m_sDBUser}',\n");
 			fwrite($hFile, "\t'db_pwd' => '".addslashes($this->m_sDBPwd)."',\n");

core/dbobject.class.php

@@ -43,7 +43,9 @@ abstract class DBObject
 										// The object may have incorrect external keys, then any attempt of reload must be avoided
 	private $m_bCheckStatus = null; // Means: the object has been verified and is consistent with integrity rules
 													//        if null, then the check has to be performed again to know the status
-													//        otherwise, 
+	protected $m_aCheckIssues = null;
+	protected $m_aAsArgs = null; // The current object as a standard argument (cache)
+
 	private $m_bFullyLoaded = false; // Compound objects can be partially loaded
 	private $m_aLoadedAtt = array(); // Compound objects can be partially loaded, array of sAttCode
 
@@ -277,7 +279,10 @@ abstract class DBObject
 			}
 			else
 			{
+				// The object has changed, reset caches
 				$this->m_bCheckStatus = null;
+				$this->m_aAsArgs = null;
+
 				$this->m_aCurrValues[$sAttCode] = $value->GetKey();
 				foreach(MetaModel::ListAttributeDefs(get_class($this)) as $sCode => $oDef)
 				{
@@ -313,8 +318,12 @@ abstract class DBObject
 		$realvalue = $oAttDef->MakeRealValue($value);
 		$this->m_aCurrValues[$sAttCode] = $realvalue;
 
+		// The object has changed, reset caches
 		$this->m_bCheckStatus = null;
-		$this->RegisterAsDirty(); // Make sure we do not reload it anymore... before saving it
+		$this->m_aAsArgs = null;
+
+		// Make sure we do not reload it anymore... before saving it
+		$this->RegisterAsDirty();
 	}
 
 	public function Get($sAttCode)
@@ -619,11 +628,14 @@ abstract class DBObject
 		}
 		elseif ($oAtt->IsExternalKey())
 		{
-			$sTargetClass = $oAtt->GetTargetClass();
-			$oTargetObj = MetaModel::GetObject($sTargetClass, $toCheck, false /*must be found*/, true /*allow all data*/);
-			if (is_null($oTargetObj))
+			if (!MetaModel::SkipCheckExtKeys())
 			{
-				return "Target object not found ($sTargetClass::$toCheck)";
+				$sTargetClass = $oAtt->GetTargetClass();
+				$oTargetObj = MetaModel::GetObject($sTargetClass, $toCheck, false /*must be found*/, true /*allow all data*/);
+				if (is_null($oTargetObj))
+				{
+					return "Target object not found ($sTargetClass::$toCheck)";
+				}
 			}
 		}
 		elseif ($oAtt->IsScalar())
@@ -689,14 +701,15 @@ abstract class DBObject
 
 	final public function CheckToWrite()
 	{
-		if (false)
+		if (MetaModel::SkipCheckToWrite())
 		{
 			return array(true, array());
 		}
-
 		if (is_null($this->m_bCheckStatus))
 		{
+			$oKPI = new ExecutionKPI();
 			$this->DoCheckToWrite();
+			$oKPI->ComputeStats('CheckToWrite', get_class($this));
 			if (count($this->m_aCheckIssues) == 0)
 			{
 				$this->m_bCheckStatus = true;
@@ -1106,24 +1119,32 @@ abstract class DBObject
 	//       2) set only the object ref and resolve the values iif needed from contextual templates and queries (easy for the queries, not for the templates)
 	public function ToArgs($sArgName = 'this')
 	{
-		$aScalarArgs = array();
-		$aScalarArgs[$sArgName] = $this->GetKey();
-		$aScalarArgs[$sArgName.'->id'] = $this->GetKey();
-		$aScalarArgs[$sArgName.'->object()'] = $this;
-		$aScalarArgs[$sArgName.'->hyperlink()'] = $this->GetHyperlink();
-		$aScalarArgs[$sArgName.'->name()'] = $this->GetName();
-	
-		$sClass = get_class($this);
-		foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
-		{
-			$aScalarArgs[$sArgName.'->'.$sAttCode] = $this->Get($sAttCode);
-			if ($oAttDef->IsScalar())
+		if (is_null($this->m_aAsArgs))
+		{
+			$oKPI = new ExecutionKPI();
+			$aScalarArgs = array();
+			$aScalarArgs[$sArgName] = $this->GetKey();
+			$aScalarArgs[$sArgName.'->id'] = $this->GetKey();
+			$aScalarArgs[$sArgName.'->object()'] = $this;
+			$aScalarArgs[$sArgName.'->hyperlink()'] = $this->GetHyperlink();
+			$aScalarArgs[$sArgName.'->name()'] = $this->GetName();
+		
+			$sClass = get_class($this);
+			foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
 			{
-				$aScalarArgs[$sArgName.'->html('.$sAttCode.')'] = $this->GetAsHtml($sAttCode);
-				$aScalarArgs[$sArgName.'->label('.$sAttCode.')'] = strip_tags($this->GetAsHtml($sAttCode));
+				$aScalarArgs[$sArgName.'->'.$sAttCode] = $this->Get($sAttCode);
+				if ($oAttDef->IsScalar())
+				{
+					// #@# Note: This has been proven to be quite slow, this can slow down bulk load
+					$sAsHtml = $this->GetAsHtml($sAttCode);
+					$aScalarArgs[$sArgName.'->html('.$sAttCode.')'] = $sAsHtml;
+					$aScalarArgs[$sArgName.'->label('.$sAttCode.')'] = strip_tags($sAsHtml);
+				}
 			}
+			$this->m_aAsArgs = $aScalarArgs;
+			$oKPI->ComputeStats('ToArgs', get_class($this));
 		}
-		return $aScalarArgs;
+		return $this->m_aAsArgs;
 	}
 
 	// To be optionaly overloaded

core/metamodel.class.php

@@ -202,6 +202,9 @@ abstract class MetaModel
 
 	private static $m_oConfig = null;
 
+	private static $m_bSkipCheckToWrite = false;
+	private static $m_bSkipCheckExtKeys = false;
+
 	private static $m_bQueryCacheEnabled = false;
 	private static $m_bTraceQueries = false;
 	private static $m_aQueriesLog = array();
@@ -210,6 +213,16 @@ abstract class MetaModel
 	private static $m_bLogNotification = false;
 	private static $m_bLogWebService = false;
 
+	public static function SkipCheckToWrite()
+	{
+		return self::$m_bSkipCheckToWrite;
+	}
+
+	public static function SkipCheckExtKeys()
+	{
+		return self::$m_bSkipCheckExtKeys;
+	}
+
 	public static function IsLogEnabledIssue()
 	{
 		return self::$m_bLogIssue;
@@ -3254,6 +3267,9 @@ abstract class MetaModel
 		self::$m_bTraceQueries = self::$m_oConfig->GetDebugQueries();
 		self::$m_bQueryCacheEnabled = self::$m_oConfig->GetQueryCacheEnabled();
 
+		self::$m_bSkipCheckToWrite = self::$m_oConfig->Get('skip_check_to_write');
+		self::$m_bSkipCheckExtKeys = self::$m_oConfig->Get('skip_check_ext_keys');
+
 		// Note: load the dictionary as soon as possible, because it might be
 		//       needed when some error occur
 		foreach (self::$m_oConfig->GetDictionaries() as $sModule => $sToInclude)

setup/benchmark.php

@@ -859,7 +859,6 @@ try
 
 		default:
 		$oP->error("Error: unsupported operation '$sOperation'");
-		
 	}
 }
 catch(ZZException $e)
@@ -871,5 +870,7 @@ catch(ZZCoreException $e)
 	$oP->error("Error: '".$e->getHtmlDesc()."'");	
 }
 $oKPI->ComputeAndReport('Total execution');
+// too big (showing all queries) ExecutionKPI::ReportStats();
+//MetaModel::ShowQueryTrace();
 $oP->output();
 ?>