#111 Improved the data loader, and added a REST service to load data from a file

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@892 a333f486-631f-4898-b8df-5754b55c2be0

core/dbobject.class.php

@@ -669,6 +669,8 @@ abstract class DBObject
 	// a displayable error is returned
 	public function DoCheckToWrite()
 	{
+		$this->DoComputeValues();
+
 		$this->m_aCheckIssues = array();
 
 		foreach(MetaModel::ListAttributeDefs(get_class($this)) as $sAttCode=>$oAttDef)

setup/xmldataloader.class.inc.php

@@ -41,7 +41,11 @@ class XMLDataLoader
 	protected $m_bSessionActive;
 	protected $m_oChange;
 	protected $m_sCacheFileName;
-	
+
+	protected $m_aErrors;
+	protected $m_aWarnings;
+	protected $m_iCountCreated;
+
 	public function __construct($sConfigFileName)
 	{
 		$this->m_aKeys = array();
@@ -51,7 +55,9 @@ class XMLDataLoader
 		$this->InitDataModel($sConfigFileName);
 		$this->LoadKeysCache();
 		$this->m_bSessionActive = true;
-
+		$this->m_aErrors = array();
+		$this->m_aWarnings = array();
+		$this->m_iCountCreated = 0;
 	}
 	
 	public function StartSession($oChange)
@@ -63,10 +69,38 @@ class XMLDataLoader
 		$this->m_bSessionActive  = true;
 	}
 	
-	public function EndSession()
+	public function EndSession($bStrict = false)
 	{
 		$this->ResolveExternalKeys();
 		$this->m_bSessionActive  = false;
+
+		if (count($this->m_aErrors) > 0)
+		{
+			return false;
+		}
+		elseif ($bStrict && count($this->m_aWarnings) > 0)
+		{
+			return false;
+		}
+		else
+		{
+			return true;
+		}
+	}
+
+	public function GetErrors()
+	{
+		return $this->m_aErrors;
+	}
+
+	public function GetWarnings()
+	{
+		return $this->m_aWarnings;
+	}
+
+	public function GetCountCreated()
+	{
+		return $this->m_iCountCreated;
 	}
 	
 	public function __destruct()
@@ -116,7 +150,10 @@ class XMLDataLoader
 		{
 			$sData = serialize( array('keys' => $this->m_aKeys,
 									'objects' => $this->m_aObjectsCache,
-									'change' => $this->m_oChange));
+									'change' => $this->m_oChange,
+									'errors' => $this->m_aErrors,
+									'warnings' => $this->m_aWarnings,
+									));
 			fwrite($hFile, $sData);
 			fclose($hFile);
 		}
@@ -137,7 +174,9 @@ class XMLDataLoader
 			$aCache = unserialize($sFileContent);
 			$this->m_aKeys = $aCache['keys'];
 			$this->m_aObjectsCache = $aCache['objects']; 
-			$this->m_oChange = $aCache['change']; 
+			$this->m_oChange = $aCache['change'];
+			$this->m_aErrors = $aCache['errors'];
+			$this->m_aWarnings = $aCache['warnings'];
 		}
 	}	 	
 	
@@ -170,6 +209,12 @@ class XMLDataLoader
 		$aReplicas  = array();
 		foreach($oXml as $sClass => $oXmlObj)
 		{
+			if (!MetaModel::IsValidClass($sClass))
+			{
+				SetupWebPage::log_error("Unknown class - $sClass");
+				throw(new Exception("Unknown class - $sClass"));
+			}
+
 			$iSrcId = (integer)$oXmlObj['id']; // Mandatory to cast
 			
 			// Import algorithm
@@ -177,40 +222,84 @@ class XMLDataLoader
 			// for all attribute that is neither an external field
 			// not an external key, assign it
 			// Store all external keys for further reference
-			// Create the object an store the correspondence between its newly created Id
+			// Create the object an store the correspondance between its newly created Id
 			// and its original Id
 			// Once all the objects have been created re-assign all the external keys to
 			// their actual Ids
 			$oTargetObj = MetaModel::NewObject($sClass);
-			foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode=>$oAttDef)
+			foreach($oXmlObj as $sAttCode => $oSubNode)
 			{
+				if (!MetaModel::IsValidAttCode($sClass, $sAttCode))
+				{
+					$sMsg = "Unknown attribute code - $sClass/$sAttCode";
+					SetupWebPage::log_error($sMsg);
+					throw(new Exception($sMsg));
+				}
+
+				$oAttDef = MetaModel::GetAttributeDef($sClass, $sAttCode);
 				if (($oAttDef->IsWritable()) && ($oAttDef->IsScalar()))
 				{
 					if ($oAttDef->IsExternalKey())
 					{
-						$iDstObj = (integer)($oXmlObj->$sAttCode);
-						// Attempt to find the object in the list of loaded objects
-						$iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj);
-						if ($iExtKey == 0)
+						if (substr(trim($oSubNode), 0, 6) == 'SELECT')
 						{
-							$iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative !
-							$oTargetObj->RegisterAsDirty();
+							$sQuery = trim($oSubNode);
+							$oSet = new DBObjectSet(DBObjectSearch::FromOQL($sQuery));
+							$iMatches = $oSet->Count();
+							if ($iMatches == 1)
+							{
+								$oFoundObject = $oSet->Fetch();
+								$iExtKey = $oFoundObject->GetKey();
+							}
+							else
+							{
+								$sMsg = "Ext key not reconcilied - $sClass/$iSrcId - $sAttCode: '".$sQuery."' - found $iMatches matche(s)";
+								SetupWebPage::log_error($sMsg);
+								$this->m_aErrors[] = $sMsg;
+								$iExtKey = 0;
+							}
+						}
+						else
+						{
+							$iDstObj = (integer)($oSubNode);
+							// Attempt to find the object in the list of loaded objects
+							$iExtKey = $this->GetObjectKey($oAttDef->GetTargetClass(), $iDstObj);
+							if ($iExtKey == 0)
+							{
+								$iExtKey = -$iDstObj; // Convention: Unresolved keys are stored as negative !
+								$oTargetObj->RegisterAsDirty();
+							}
+							// here we allow external keys to be invalid because we will resolve them later on...
 						}
-						// here we allow external keys to be invalid because we will resolve them later on...
 						//$oTargetObj->CheckValue($sAttCode, $iExtKey);
 						$oTargetObj->Set($sAttCode, $iExtKey);
 					}
+					elseif ($oAttDef instanceof AttributeBlob)
+					{
+						$sMimeType = (string) $oSubNode->mimetype;
+						$sFileName = (string) $oSubNode->filename;
+						$data = base64_decode((string) $oSubNode->data);
+						$oDoc = new ormDocument($data, $sMimeType, $sFileName);
+						$oTargetObj->Set($sAttCode, $oDoc);
+					}
 					else
 					{
-						// tested by Romain, little impact on perf (not significant on the intial setup)
-						$res = $oTargetObj->CheckValue($sAttCode, (string)$oXmlObj->$sAttCode);
+						$value = (string)$oSubNode;
+
+						if ($value == '')
+						{
+							$value = $oAttDef->GetNullValue();
+						}
+
+						$res = $oTargetObj->CheckValue($sAttCode, $value);
 						if ($res !== true)
 						{
 							// $res contains the error description
-							SetupWebPage::log_error("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res");
-							throw(new Exception("Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oXmlObj->$sAttCode."' ; $res"));
+							$sMsg = "Value not allowed - $sClass/$iSrcId - $sAttCode: '".$oSubNode."' ; $res";
+							SetupWebPage::log_error($sMsg);
+							$this->m_aErrors[] = $sMsg;
 						}
-						$oTargetObj->Set($sAttCode, (string)$oXmlObj->$sAttCode);
+						$oTargetObj->Set($sAttCode, $value);
 					}
 				}
 			}
@@ -283,12 +372,13 @@ class XMLDataLoader
 				{
 			        $iObjId = $oTargetObj->DBInsertNoReload();
 				}
+				$this->m_iCountCreated++;
 			}	        
 		}
 		catch(Exception $e)
 		{
-			SetupWebPage::log_error("An object could not be loaded - $sClass/$iSrcId - ".$e->getMessage());
-			echo $e->GetHtmlDesc();
+			SetupWebPage::log_error("An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage());
+			$this->m_aErrors[] = "An object could not be recorded - $sClass/$iSrcId - ".$e->getMessage();
 		}
 		$aParentClasses = MetaModel::EnumParentClasses($sClass);
 		$aParentClasses[] = $sClass;
@@ -323,6 +413,7 @@ class XMLDataLoader
 						{
 							$sMsg = "unresolved extkey in $sClass::".$oTargetObj->GetKey()."(".$oTargetObj->GetName().")::$sAttCode=$sTargetClass::$iTempKey";
 							SetupWebPage::log_warning($sMsg);
+							$this->m_aWarnings[] = $sMsg;
 							//echo "<pre>aKeys[".$sTargetClass."]:\n";
 							//print_r($this->m_aKeys[$sTargetClass]);
 							//echo "</pre>\n";
@@ -349,7 +440,7 @@ class XMLDataLoader
 					}
 					catch(Exception $e)
 					{
-						echo $e->GetHtmlDesc();
+						$this->m_aErrors[] = "The object changes could not be tracked - $sClass/$iSrcId - ".$e->getMessage();
 					}
 				}
 			}

webservices/backoffice.dataloader.php

@@ -0,0 +1,172 @@
+<?php
+// Copyright (C) 2010 Combodo SARL
+//
+//   This program is free software; you can redistribute it and/or modify
+//   it under the terms of the GNU General Public License as published by
+//   the Free Software Foundation; version 3 of the License.
+//
+//   This program is distributed in the hope that it will be useful,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+//   GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program; if not, write to the Free Software
+//   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+/**
+ * Does load data from XML files (currently used in the setup and the backoffice data loader utility)
+ *
+ * @author      Erwan Taloc <erwan.taloc@combodo.com>
+ * @author      Romain Quetiez <romain.quetiez@combodo.com>
+ * @author      Denis Flaven <denis.flaven@combodo.com>
+ * @license     http://www.opensource.org/licenses/gpl-3.0.html LGPL
+ */
+
+/**
+ * This page is called to load an XML file into the database
+ * parameters
+ * 'file' string Name of the file to load
+ */ 
+define('SAFE_MINIMUM_MEMORY', 256*1024*1024);
+require_once('../application/utils.inc.php');
+require_once("../application/nicewebpage.class.inc.php");
+// required because the class xmldataloader is reporting errors in the setup.log file
+require_once('../setup/setuppage.class.inc.php');
+
+
+function SetMemoryLimit($oP)
+{
+	$sMemoryLimit = trim(ini_get('memory_limit'));
+	if (empty($sMemoryLimit))
+	{
+		// On some PHP installations, memory_limit does not exist as a PHP setting!
+		// (encountered on a 5.2.0 under Windows)
+		// In that case, ini_set will not work, let's keep track of this and proceed with the data load
+		$oP->p("No memory limit has been defined in this instance of PHP");		
+	}
+	else
+	{
+		// Check that the limit will allow us to load the data
+		//
+		$iMemoryLimit = utils::ConvertToBytes($sMemoryLimit);
+		if ($iMemoryLimit < SAFE_MINIMUM_MEMORY)
+		{
+			if (ini_set('memory_limit', SAFE_MINIMUM_MEMORY) === FALSE)
+			{
+				$oP->p("memory_limit is too small: $iMemoryLimit and can not be increased by the script itself.");		
+			}
+			else
+			{
+				$oP->p("memory_limit increased from $iMemoryLimit to ".SAFE_MINIMUM_MEMORY.".");		
+			}
+		}
+	}
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// Main
+//
+////////////////////////////////////////////////////////////////////////////////
+
+require_once('../core/config.class.inc.php');
+require_once('../core/log.class.inc.php');
+require_once('../core/kpi.class.inc.php');
+require_once('../core/cmdbsource.class.inc.php');
+require_once('../setup/xmldataloader.class.inc.php');
+
+define('FINAL_CONFIG_FILE', '../config-itop.php');
+
+// Never cache this page
+header("Cache-Control: no-cache, must-revalidate");  // HTTP/1.1
+header("Expires: Fri, 17 Jul 1970 05:00:00 GMT");    // Date in the past
+
+Utils::SpecifyConfigFile(FINAL_CONFIG_FILE);
+
+/**
+ * Main program
+ */
+$sFileName = Utils::ReadParam('file', '');
+
+$oP = new WebPage("iTop - Backoffice data loader");
+
+
+try
+{
+	// Note: the data model must be loaded first
+	$oDataLoader = new XMLDataLoader(FINAL_CONFIG_FILE); // When called by the wizard, the final config is not yet there
+
+	if (empty($sFileName))
+	{
+		throw(new Exception("Missing argument 'file'"));
+	}
+	if (!file_exists($sFileName))
+	{
+		throw(new Exception("File $sFileName does not exist"));
+	}
+
+	SetMemoryLimit($oP);
+	
+
+	// The XMLDataLoader constructor has initialized the DB, let's start a transaction 
+	CMDBSource::Query('SET AUTOCOMMIT=0');
+	CMDBSource::Query('BEGIN WORK');
+	
+	$oChange = MetaModel::NewObject("CMDBChange");
+	$oChange->Set("date", time());
+	$oChange->Set("userinfo", "Initialization");
+	$iChangeId = $oChange->DBInsert();
+	$oP->p("Starting data load.");		
+	$oDataLoader->StartSession($oChange);
+	
+	$oDataLoader->LoadFile($sFileName);
+	
+	$oP->p("Ending data load session");
+	if ($oDataLoader->EndSession(true /* strict */))
+	{
+		$iCountCreated = $oDataLoader->GetCountCreated();
+		CMDBSource::Query('COMMIT');
+
+		$oP->p("Data successfully written into the DB: $iCountCreated objects created");
+	}
+	else
+	{
+		CMDBSource::Query('ROLLBACK');
+		$oP->p("Some issues have been encountered, changes will not be recorded, please review the source data");
+		$aErrors = $oDataLoader->GetErrors();
+		if (count($aErrors) > 0)
+		{
+			$oP->p('Errors ('.count($aErrors).')');
+			foreach ($aErrors as $sMsg)
+			{
+				$oP->p(' * '.$sMsg);
+			}
+		}
+		$aWarnings = $oDataLoader->GetWarnings();
+		if (count($aWarnings) > 0)
+		{
+			$oP->p('Warnings ('.count($aWarnings).')');
+			foreach ($aWarnings as $sMsg)
+			{
+				$oP->p(' * '.$sMsg);
+			}
+		}
+	}
+
+}
+catch(Exception $e)
+{
+	$oP->p("An error happened while loading the data: ".$e->getMessage());		
+	$oP->p("Aborting (no data written)...");		
+	CMDBSource::Query('ROLLBACK');
+}
+
+if (function_exists('memory_get_peak_usage'))
+{
+	$oP->p("Information: memory peak usage: ".memory_get_peak_usage());
+}
+
+$oP->Output();
+?>