Data Exchange - Implemented reconciliation on external keys

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@1101 a333f486-631f-4898-b8df-5754b55c2be0

core/csvparser.class.inc.php

@@ -40,6 +40,8 @@ define('evTEXTQUAL', 3); // used for escaping as well
 define('evOTHERCHAR', 4);
 define('evEND', 5);
 
+define('NULL_VALUE', '<NULL>');
+
 
 /**
  * CSVParser
@@ -82,6 +84,10 @@ class CSVParser
 		{
 			$sCell = $this->m_sCurrCell;
 		}
+		if ($sCell == NULL_VALUE)
+		{
+			$sCell = null;
+		}
 
 		if (!is_null($aFieldMap))
 		{

core/metamodel.class.php

@@ -3730,6 +3730,31 @@ abstract class MetaModel
 		return $oObj;
 	}
 
+	static protected $m_aCacheObjectByColumn = array();
+
+	public static function GetObjectByColumn($sClass, $sAttCode, $value, $bMustBeFoundUnique = true)
+	{
+		if (!isset(self::$m_aCacheObjectByColumn[$sClass][$sAttCode][$value]))
+		{
+			self::_check_subclass($sClass);	
+	
+			$oObjSearch = new DBObjectSearch($sClass);
+			$oObjSearch->AddCondition($sAttCode, $value, '=');
+			$oSet = new DBObjectSet($oObjSearch);
+			if ($oSet->Count() == 1)
+			{
+				self::$m_aCacheObjectByColumn[$sClass][$sAttCode][$value] = $oSet->fetch();
+			}
+			else
+			{
+				if ($bMustBeFoundUnique) throw new CoreException('Failed to get an object by column', array('class'=>$sClass, 'attcode'=>$sAttCode, 'value'=>$value, 'matches' => $oSet->Count()));
+				self::$m_aCacheObjectByColumn[$sClass][$sAttCode][$value] = null;
+			}
+		}
+
+		return self::$m_aCacheObjectByColumn[$sClass][$sAttCode][$value];
+	}
+
 	public static function GetObjectFromOQL($sQuery, $aParams = null, $bAllowAllData = false)
 	{
 		$oFilter = DBObjectSearch::FromOQL($sQuery, $aParams);

synchro/synchro_import.php

@@ -409,7 +409,14 @@ try
 				$aValues = array(); // Used to build the insert query
 				foreach ($aRow as $iCol => $value)
 				{
-					$aValues[] = CMDBSource::Quote($value);
+					if (is_null($value))
+					{
+						$aValues[] = 'NULL';
+					}
+					else
+					{
+						$aValues[] = CMDBSource::Quote($value);
+					}
 				}
 				$sValues = implode(', ', $aValues);
 				$sInsert = "INSERT INTO `$sTable` ($sInsertColumns) VALUES ($sValues)";

synchro/synchrodatasource.class.inc.php

@@ -53,6 +53,8 @@ class SynchroDataSource extends cmdbAbstractObject
 		MetaModel::Init_AddAttribute(new AttributeEnum("status", array("allowed_values"=>new ValueSetEnum('implementation,production,obsolete'), "sql"=>"status", "default_value"=>"implementation", "is_null_allowed"=>false, "depends_on"=>array())));
 		MetaModel::Init_AddAttribute(new AttributeExternalKey("user_id", array("targetclass"=>"User", "jointype"=>null, "allowed_values"=>null, "sql"=>"user_id", "is_null_allowed"=>true, "on_target_delete"=>DEL_MANUAL, "depends_on"=>array())));
 		MetaModel::Init_AddAttribute(new AttributeClass("scope_class", array("class_category"=>"bizmodel", "more_values"=>"", "sql"=>"scope_class", "default_value"=>null, "is_null_allowed"=>false, "depends_on"=>array())));
+		
+		// Declared here for a future usage, but ignored so far
 		MetaModel::Init_AddAttribute(new AttributeString("scope_restriction", array("allowed_values"=>null, "sql"=>"scope_restriction", "default_value"=>null, "is_null_allowed"=>true, "depends_on"=>array())));
 		
 		//MetaModel::Init_AddAttribute(new AttributeDateTime("last_synchro_date", array("allowed_values"=>null, "sql"=>"last_synchro_date", "default_value"=>"", "is_null_allowed"=>false, "depends_on"=>array())));
@@ -76,7 +78,7 @@ class SynchroDataSource extends cmdbAbstractObject
 		MetaModel::Init_AddAttribute(new AttributeLinkedSet("status_list", array("linked_class"=>"SynchroLog", "ext_key_to_me"=>"sync_source_id", "allowed_values"=>null, "count_min"=>0, "count_max"=>0, "depends_on"=>array())));
 
 		// Display lists
-		MetaModel::Init_SetZListItems('details', array('name', 'description', 'scope_class', 'scope_restriction', 'status', 'user_id', 'full_load_periodicity', 'reconciliation_policy', 'action_on_zero', 'action_on_one', 'action_on_multiple', 'delete_policy', 'delete_policy_update', 'delete_policy_retention' /*'attribute_list'*/, 'status_list')); // Attributes to be displayed for the complete details
+		MetaModel::Init_SetZListItems('details', array('name', 'description', 'scope_class', /*'scope_restriction', */'status', 'user_id', 'full_load_periodicity', 'reconciliation_policy', 'action_on_zero', 'action_on_one', 'action_on_multiple', 'delete_policy', 'delete_policy_update', 'delete_policy_retention' /*'attribute_list'*/, 'status_list')); // Attributes to be displayed for the complete details
 		MetaModel::Init_SetZListItems('list', array('scope_class', 'status', 'user_id', 'full_load_periodicity')); // Attributes to be displayed for a list
 		// Search criteria
 		MetaModel::Init_SetZListItems('standard_search', array('name', 'status', 'scope_class', 'user_id')); // Criteria of the std search form
@@ -112,7 +114,15 @@ class SynchroDataSource extends cmdbAbstractObject
 					}
 					else
 					{
-						$oAttribute = new SynchroAttribute();
+						if ($oAttDef->IsExternalKey())
+						{
+							$oAttribute = new SynchroAttExtKey();
+							$oAttribute->Set('reconciliation_attcode', ''); // Blank means by pkey
+						}
+						else
+						{
+							$oAttribute = new SynchroAttribute();
+						}
 						$oAttribute->Set('sync_source_id', $this->GetKey());
 						$oAttribute->Set('attcode', $sAttCode);
 						$oAttribute->Set('reconcile', MetaModel::IsReconcKey($this->GetTargetClass(), $sAttCode) ? 1 : 0);
@@ -327,7 +337,16 @@ EOF
 		{
 			if(!isset($aAttributes[$sAttCode]))
 			{
-				$oAttribute = new SynchroAttribute();
+				$oAttDef = MetaModel::GetAttributeDef($this->GetTargetClass(), $sAttCode);
+				if ($oAttDef->IsExternalKey())
+				{
+					$oAttribute = new SynchroAttExtKey();
+					$oAttribute->Set('reconciliation_attcode', ''); // Blank means by pkey
+				}
+				else
+				{
+					$oAttribute = new SynchroAttribute();
+				}
 				$oAttribute->Set('sync_source_id', $this->GetKey());
 				$oAttribute->Set('attcode', $sAttCode);
 			}
@@ -419,7 +438,16 @@ EOF
 		{
 			if ($oAttDef->IsScalar() && $oAttDef->IsWritable())
 			{
-				$oAttribute = new SynchroAttribute();
+				$oAttDef = MetaModel::GetAttributeDef($this->GetTargetClass(), $sAttCode);
+				if ($oAttDef->IsExternalKey())
+				{
+					$oAttribute = new SynchroAttExtKey();
+					$oAttribute->Set('reconciliation_attcode', ''); // Blank means by pkey
+				}
+				else
+				{
+					$oAttribute = new SynchroAttribute();
+				}
 				$oAttribute->Set('sync_source_id', $this->GetKey());
 				$oAttribute->Set('attcode', $sAttCode);
 				$oAttribute->Set('reconcile', MetaModel::IsReconcKey($this->GetTargetClass(), $sAttCode) ? 1 : 0);
@@ -668,15 +696,15 @@ EOF
 		{
 			if ($oSyncAtt->Get('update'))
 			{
-				$aAttCodesToUpdate[] = $oSyncAtt->Get('attcode');
+				$aAttCodesToUpdate[$oSyncAtt->Get('attcode')] = $oSyncAtt;
 			}
 			if ($oSyncAtt->Get('reconcile'))
 			{
-				$aAttCodesToReconcile[] = $oSyncAtt->Get('attcode');
+				$aAttCodesToReconcile[$oSyncAtt->Get('attcode')] = $oSyncAtt;
 			}
-			$aAttCodesExpected[] = $oSyncAtt->Get('attcode');
+			$aAttCodesExpected[$oSyncAtt->Get('attcode')] = $oSyncAtt;
 		}
-		$aColumns = $this->GetSQLColumns($aAttCodesExpected);
+		$aColumns = $this->GetSQLColumns(array_keys($aAttCodesExpected));
 		$aExtDataFields = array_keys($aColumns);
 		$aExtDataFields[] = 'primary_key';
 		$aExtDataSpec = array(
@@ -693,17 +721,19 @@ EOF
 		elseif ($this->Get('reconciliation_policy') == 'use_primary_key')
 		{
 			// Override the setings made at the attribute level !
-			$aReconciliationKeys = array("primary_key");
+			$aReconciliationKeys = array("primary_key" => null);
 		}
-		$aTraces[] = "Reconciliation on: {".implode(', ', $aReconciliationKeys)."}";
+
+		$aTraces[] = "Update of: {".implode(', ', array_keys($aAttCodesToUpdate))."}";
+		$aTraces[] = "Reconciliation on: {".implode(', ', array_keys($aReconciliationKeys))."}";
 		
 		$aAttributes = array();
-		foreach($aAttCodesToUpdate as $sAttCode)
+		foreach($aAttCodesToUpdate as $sAttCode => $oSyncAtt)
 		{
 			$oAttDef = MetaModel::GetAttributeDef($this->GetTargetClass(), $sAttCode);
 			if ($oAttDef->IsWritable() && $oAttDef->IsScalar())
 			{
-				$aAttributes[] = $sAttCode;
+				$aAttributes[$sAttCode] = $oSyncAtt;
 			}
 		}
 		
@@ -761,9 +791,17 @@ EOF
 		{
 			$oAttDef = MetaModel::GetAttributeDef($sClass, $sAttCode);
 			
-			foreach($oAttDef->GetSQLColumns() as $sField => $sDBFieldType)
+			if ($oAttDef->IsExternalKey())
+			{
+				// The pkey might be used as well as any other key column
+				$aColumns[$sAttCode] = 'VARCHAR (255)';
+			}
+			else
 			{
-				$aColumns[$sField] = $sDBFieldType;
+				foreach($oAttDef->GetSQLColumns() as $sField => $sDBFieldType)
+				{
+					$aColumns[$sField] = $sDBFieldType;
+				}
 			}
 		}
 		return $aColumns;
@@ -1068,7 +1106,7 @@ class SynchroReplica extends DBObject
 			// If needed, construct the query used for the reconciliation
 			if (!isset(self::$aSearches[$oDataSource->GetKey()]))
 			{
-				foreach($aReconciliationKeys as $sFilterCode)
+				foreach($aReconciliationKeys as $sFilterCode => $oSyncAtt)
 				{
 					$aCriterias[] = ($sFilterCode == 'primary_key' ? 'id' : $sFilterCode).' = :'.$sFilterCode;
 				}
@@ -1077,9 +1115,9 @@ class SynchroReplica extends DBObject
 			}
 			// Get the criterias for the search
 			$aFilterValues = array();
-			foreach($aReconciliationKeys as $sFilterCode)
+			foreach($aReconciliationKeys as $sFilterCode => $oSyncAtt)
 			{
-				$value = $this->GetValueFromExtData($sFilterCode);
+				$value = $this->GetValueFromExtData($sFilterCode, $oSyncAtt, $oStatLog, $aTraces);
 				if (!is_null($value))
 				{
 					$aFilterValues[$sFilterCode] = $value;
@@ -1184,9 +1222,9 @@ class SynchroReplica extends DBObject
 	protected function UpdateObjectFromReplica($oDestObj, $aAttributes, $oChange, &$oStatLog, &$aTraces, $sStatsCode, $sStatsCodeError)
 	{
 		$aValueTrace = array();
-		foreach($aAttributes as $sAttCode)
+		foreach($aAttributes as $sAttCode => $oSyncAtt)
 		{
-			$value = $this->GetValueFromExtData($sAttCode);
+			$value = $this->GetValueFromExtData($sAttCode, $oSyncAtt, $oStatLog, $aTraces);
 			if (!is_null($value))
 			{
 				$oDestObj->Set($sAttCode, $value);
@@ -1228,9 +1266,9 @@ class SynchroReplica extends DBObject
 		try
 		{
 			$aValueTrace = array();
-			foreach($aAttributes as $sAttCode)
+			foreach($aAttributes as $sAttCode => $oSyncAtt)
 			{
-				$value = $this->GetValueFromExtData($sAttCode);
+				$value = $this->GetValueFromExtData($sAttCode, $oSyncAtt, $oStatLog, $aTraces);
 				if (!is_null($value))
 				{
 					$oDestObj->Set($sAttCode, $value);
@@ -1319,10 +1357,41 @@ class SynchroReplica extends DBObject
 	/**
 	 * Get the value from the 'Extended Data' located in the synchro_data_xxx table for this replica
 	 */
-	 protected function GetValueFromExtData($sColumnName)
+	 protected function GetValueFromExtData($sColumnName, $oSyncAtt, &$oStatLog, &$aTraces)
 	 {
-	 	// $aData should contain attributes defined either for reconciliation or update
-	 	$aData = $this->GetExtendedData();
+	 	// $aData should contain attributes defined either for reconciliation or create/update
+		$aData = $this->GetExtendedData();
+
+      // In any case, a null column means "ignore this column"
+      //
+		if (is_null($aData[$sColumnName]))
+      {
+      	return null;
+		}
+
+		if (!is_null($oSyncAtt) && ($oSyncAtt instanceof SynchroAttExtKey))
+		{
+			$sReconcAttCode = $oSyncAtt->Get('reconciliation_attcode');
+			if (!empty($sReconcAttCode))
+			{
+				$oDataSource = MetaModel::GetObject('SynchroDataSource', $this->Get('sync_source_id'));
+			 	$sClass = $oDataSource->GetTargetClass();
+		 		$oAttDef = MetaModel::GetAttributeDef($sClass, $sColumnName);
+		 		$sRemoteClass = $oAttDef->GetTargetClass();
+				$oObj = MetaModel::GetObjectByColumn($sRemoteClass, $sReconcAttCode, $aData[$sColumnName], false);
+		 		if ($oObj)
+		 		{
+					 return $oObj->GetKey();
+				}
+				else
+				{
+					// Note: differs from null (in which case the value would be left unchanged)
+					$aTraces[] = "Could not find [unique] object for '$sColumnName': searched on $sReconcAttCode = '$aData[$sColumnName]'";
+					return 0;
+				}
+			}
+		}
+
  		return $aData[$sColumnName];
 	 }
 }

test/test.class.inc.php

@@ -413,13 +413,26 @@ abstract class TestBizModel extends TestHandler
 	}
 
 	protected $m_oChange;
-	protected function ObjectToDB($oNew, $bReload = false)
+	protected function GetCurrentChange()
 	{
-		list($bRes, $aIssues) = $oNew->CheckToWrite();
-		if (!$bRes)
+		if (!isset($this->m_oChange))
 		{
-			throw new CoreException('Could not create object, unexpected values', array('issues' => $aIssues));
+			 new CMDBChange();
+			$oMyChange = MetaModel::NewObject("CMDBChange");
+			$oMyChange->Set("date", time());
+			$oMyChange->Set("userinfo", "Someone doing some tests");
+			$iChangeId = $oMyChange->DBInsertNoReload();
+			$this->m_oChange = $oMyChange; 
 		}
+		return $this->m_oChange;
+	}
+	protected function ObjectToDB($oNew, $bReload = false)
+	{
+//		list($bRes, $aIssues) = $oNew->CheckToWrite();
+//		if (!$bRes)
+//		{
+//			throw new CoreException('Could not create object, unexpected values', array('issues' => $aIssues));
+//		}
 		if ($oNew instanceof CMDBObject)
 		{
 			if (!isset($this->m_oChange))
@@ -431,13 +444,14 @@ abstract class TestBizModel extends TestHandler
 				$iChangeId = $oMyChange->DBInsertNoReload();
 				$this->m_oChange = $oMyChange; 
 			}
+			$oChange = $this->GetCurrentChange();
 			if ($bReload)
 			{
-				$iId = $oNew->DBInsertTracked($this->m_oChange);
+				$iId = $oNew->DBInsertTracked($oChange);
 			}
 			else
 			{
-				$iId = $oNew->DBInsertTrackedNoReload($this->m_oChange);
+				$iId = $oNew->DBInsertTrackedNoReload($oChange);
 			}
 		}
 		else
@@ -454,6 +468,18 @@ abstract class TestBizModel extends TestHandler
 		return $iId;
 	}
 
+  	protected function UpdateObjectInDB($oObject)
+	{
+   	if ($oObject instanceof CMDBObject)
+		{
+			$oChange = $this->GetCurrentChange();
+			$oObject->DBUpdateTracked($oChange);
+		}
+		else
+		{
+			$oObject->DBUpdate();
+		}
+	}
 	protected function ResetDB()
 	{
 		if (MetaModel::DBExists(false))

test/testlist.inc.php

@@ -1836,23 +1836,27 @@ class TestDataExchange extends TestBizModel
 		$oDataSource->Set('delete_policy', $aSingleScenario['delete_policy']);
 		$oDataSource->Set('delete_policy_update', $aSingleScenario['delete_policy_update']);
 		$oDataSource->Set('delete_policy_retention', $aSingleScenario['delete_policy_retention']);
-		$iDataSourceId = $this->ObjectToDB($oDataSource);
+		$iDataSourceId = $this->ObjectToDB($oDataSource, true /* reload */);
 
       $oAttributeSet = $oDataSource->Get('attribute_list');
       while ($oAttribute = $oAttributeSet->Fetch())
       {
-      	if (array_key_exists($aSingleScenario['attributes'], $oAttribute->Get('attcode')))
+      	if (array_key_exists($oAttribute->Get('attcode'), $aSingleScenario['attributes']))
       	{
       		$aAttribInfo = $aSingleScenario['attributes'][$oAttribute->Get('attcode')];
-				$oSyncAtt->Set('update', $aAttribInfo['do_update']);
-				$oSyncAtt->Set('reconcile', $aAttribInfo['do_reconcile']);
+      		if (array_key_exists('reconciliation_attcode', $aAttribInfo))
+      		{
+					$oAttribute->Set('reconciliation_attcode', $aAttribInfo['reconciliation_attcode']);
+				}
+				$oAttribute->Set('update', $aAttribInfo['do_update']);
+				$oAttribute->Set('reconcile', $aAttribInfo['do_reconcile']);
 			}
       	else
       	{
-				$oSyncAtt->Set('update', false);
-				$oSyncAtt->Set('reconcile', false);
+				$oAttribute->Set('update', false);
+				$oAttribute->Set('reconcile', false);
 			}
-			$oAttribute->DBUpdateTracked();
+			$this->UpdateObjectInDB($oAttribute);
 		}
 
 		// Prepare list of prefixes -> make sure objects are unique with regard to the reconciliation scheme
@@ -1863,7 +1867,7 @@ class TestDataExchange extends TestBizModel
 		}
 		foreach($aSingleScenario['attributes'] as $sAttCode => $aAttribInfo)
 		{
-			if ($aAttribInfo['do_reconcile'])
+			if (isset($aAttribInfo['automatic_prefix']) && $aAttribInfo['automatic_prefix'])
 			{
 				$aPrefixes[$sAttCode] = 'TEST_'.$iDataSourceId.'_';
 			}
@@ -1892,7 +1896,14 @@ class TestDataExchange extends TestBizModel
 		{
 			// Check the status (while ignoring existing objects)
 			//
-			$oObjects = new DBObjectSet(DBObjectSearch::FromOQL("SELECT $sClass WHERE id NOT IN($sExistingIds)"));
+			if (empty($sExistingIds))
+			{
+				$oObjects = new DBObjectSet(DBObjectSearch::FromOQL("SELECT $sClass"));
+			}
+			else
+			{
+				$oObjects = new DBObjectSet(DBObjectSearch::FromOQL("SELECT $sClass WHERE id NOT IN($sExistingIds)"));
+			}
 			$aFound = $oObjects->ToArray();
 			$aErrors_Unexpected = array();
 			foreach($aFound as $iObj => $oObj)
@@ -1973,8 +1984,15 @@ class TestDataExchange extends TestBizModel
 					$aFinalData = array();
 					foreach($aDataRow as $iCol => $value)
 					{
-						$sAttCode = $aSourceAttributes[$iCol];
-						$aFinalData[] = $aPrefixes[$sAttCode].$value;
+						if (is_null($value))
+						{
+							$aFinalData[] = '<NULL>';
+						}
+						else
+						{
+							$sAttCode = $aSourceAttributes[$iCol];
+							$aFinalData[] = $aPrefixes[$sAttCode].$value;
+						}
 					}
 					$sCsvData .= implode(';', $aFinalData)."\n";
 				}
@@ -2010,6 +2028,7 @@ class TestDataExchange extends TestBizModel
 				{
 					$sCsvDataViewable = $sCsvData;
 				}
+				$sCsvDataViewable = htmlentities($sCsvDataViewable);
 		
 				echo "<div style=\"\">\n";
 				echo "      <pre class=\"vardump\">$sCsvDataViewable</pre>\n";
@@ -2020,10 +2039,6 @@ class TestDataExchange extends TestBizModel
 				{
 					throw new UnitTestException('Encountered an Exception during the last import/synchro');
 				}
-				if (stripos($sRes, 'error') !== false)
-				{
-					throw new UnitTestException('Encountered an Error during the last import/synchro');
-				}
 			}
 		}
 		return;
@@ -2035,23 +2050,28 @@ class TestDataExchange extends TestBizModel
 	{
 		$aScenarios = array(
 			array(
-				'desc' => 'Simple scenario with delete option',
+				'desc' => 'Simple scenario with delete option (and extkey given as org/name)',
 				'login' => 'admin',
 				'password' => 'admin',
 				'target_class' => 'ApplicationSolution',
-				'full_load_periodicity' => '1 hour',
+				'full_load_periodicity' => 3600, // should be ignored in this case
 				'reconciliation_policy' => 'use_attributes',
 				'action_on_zero' => 'create',
 				'action_on_one' => 'update',
 				'action_on_multiple' => 'error',
-				'delete_policy' => 'update_then_delete',
-				'delete_policy_update' => 'status:obsolete',
-				'delete_policy_retention' => '',
+				'delete_policy' => 'delete',
+				'delete_policy_update' => '',
+				'delete_policy_retention' => 0,
 				'source_data' => array(
 					array('primary_key', 'org_id', 'name', 'status'),
 					array(
-						array('obj_A', 2, 'obj_A', 'production'),
-						array('obj_B', 2, 'obj_B', 'production'),
+						array('obj_A', null, 'obj_A', 'production'), // org_id unchanged
+						array('obj_B', '_DUMMY_', 'obj_B', 'production'), // error, '_DUMMY_' unknown
+						array('obj_C', 'SOMECODE', 'obj_C', 'production'),
+						array('obj_D', null, 'obj_D', 'production'),
+						array('obj_E', '_DUMMY_', 'obj_E', 'production'),
+					),
+					array(
 					),
 					array(
 					),
@@ -2061,13 +2081,22 @@ class TestDataExchange extends TestBizModel
 					array(
 						// Initial state
 						array(2, 'obj_A', 'production'),
+						array(2, 'obj_B', 'production'),
 					),
 					array(
 						array(2, 'obj_A', 'production'),
 						array(2, 'obj_B', 'production'),
+						array(1, 'obj_C', 'production'),
 					),
 					array(
-						array(2, 'obj_A', 'obsolete'),
+						array(2, 'obj_A', 'production'),
+						array(2, 'obj_B', 'production'),
+						// deleted !
+					),
+					// The only diff here is into the log
+					array(
+						array(2, 'obj_A', 'production'),
+						array(2, 'obj_B', 'production'),
 						// deleted !
 					),
 				),
@@ -2075,10 +2104,12 @@ class TestDataExchange extends TestBizModel
 					'org_id' => array(
 						'do_reconcile' => false,
 						'do_update' => true,
+						'reconciliation_attcode' => 'code',
 					),
 					'name' => array(
 						'do_reconcile' => true,
 						'do_update' => true,
+						'automatic_prefix' => true, // unique id
 					),
 					'status' => array(
 						'do_reconcile' => false,
@@ -2087,24 +2118,24 @@ class TestDataExchange extends TestBizModel
 				),
 			),
 		//);
-		//$aScenarios = array(
+		//$aXXXXScenarios = array(
 			array(
-				'desc' => 'Update then delete with retention (to complete with manual testing)',
+				'desc' => 'Update then delete with retention (to complete with manual testing) and reconciliation on org/name',
 				'login' => 'admin',
 				'password' => 'admin',
 				'target_class' => 'ApplicationSolution',
-				'full_load_periodicity' => '1 hour',
+				'full_load_periodicity' => 3600,
 				'reconciliation_policy' => 'use_attributes',
 				'action_on_zero' => 'create',
 				'action_on_one' => 'update',
 				'action_on_multiple' => 'error',
 				'delete_policy' => 'update_then_delete',
 				'delete_policy_update' => 'status:obsolete',
-				'delete_policy_retention' => '1 hour',
+				'delete_policy_retention' => 5,
 				'source_data' => array(
 					array('primary_key', 'org_id', 'name', 'status'),
 					array(
-						array('obj_A', 2, 'obj_A', 'production'),
+						array('obj_A', 'OMED', 'obj_A', 'production'),
 					),
 					array(
 					),
@@ -2124,12 +2155,14 @@ class TestDataExchange extends TestBizModel
 				),
 				'attributes' => array(
 					'org_id' => array(
-						'do_reconcile' => false,
+						'do_reconcile' => true,
 						'do_update' => true,
+						'reconciliation_attcode' => 'code',
 					),
 					'name' => array(
 						'do_reconcile' => true,
 						'do_update' => true,
+						'automatic_prefix' => true, // unique id
 					),
 					'status' => array(
 						'do_reconcile' => false,
@@ -2138,20 +2171,20 @@ class TestDataExchange extends TestBizModel
 				),
 			),
 		//);
-		//$aScenarios = array(
+		//$aXXScenarios = array(
 			array(
 				'desc' => 'Simple scenario loading a few ApplicationSolution',
 				'login' => 'admin',
 				'password' => 'admin',
 				'target_class' => 'ApplicationSolution',
-				'full_load_periodicity' => '1 hour',
+				'full_load_periodicity' => 3600,
 				'reconciliation_policy' => 'use_attributes',
 				'action_on_zero' => 'create',
 				'action_on_one' => 'update',
 				'action_on_multiple' => 'error',
 				'delete_policy' => 'update',
 				'delete_policy_update' => 'status:obsolete',
-				'delete_policy_retention' => '',
+				'delete_policy_retention' => 0,
 				'source_data' => array(
 					array('primary_key', 'org_id', 'name', 'status'),
 					array(
@@ -2161,12 +2194,20 @@ class TestDataExchange extends TestBizModel
 					),
 					array(
 						array('obj_A', 2, 'obj_A', 'production'),
+						array('obj_B', 2, 'obj_B', 'implementation'),
+						array('obj_C', 2, 'obj_C', 'implementation'),
+					),
+					array(
+						array('obj_A', 2, 'obj_A', 'production'),
 						array('obj_C', 2, 'obj_C', 'implementation'),
 						array('obj_D', 2, 'obj_D', 'implementation'),
 					),
 					array(
 						array('obj_C', 2, 'obj_C', 'production'),
 					),
+					array(
+						array('obj_C', 2, 'obj_C', 'production'),
+					),
 				),
 				'target_data' => array(
 					array('org_id', 'name', 'status'),
@@ -2187,6 +2228,12 @@ class TestDataExchange extends TestBizModel
 						array(2, 'obj_B', 'production'),
 						array(2, 'obj_B', 'implementation'),
 						array(2, 'obj_C', 'implementation'),
+					),
+					array(
+						array(2, 'obj_A', 'production'),
+						array(2, 'obj_B', 'production'),
+						array(2, 'obj_B', 'implementation'),
+						array(2, 'obj_C', 'implementation'),
 						array(2, 'obj_D', 'implementation'),
 					),
 					array(
@@ -2196,6 +2243,13 @@ class TestDataExchange extends TestBizModel
 						array(2, 'obj_C', 'production'),
 						array(2, 'obj_D', 'obsolete'),
 					),
+					array(
+						array(2, 'obj_A', 'obsolete'),
+						array(2, 'obj_B', 'production'),
+						array(2, 'obj_B', 'implementation'),
+						array(2, 'obj_C', 'production'),
+						array(2, 'obj_D', 'obsolete'),
+					),
 				),
 				'attributes' => array(
 					'org_id' => array(
@@ -2205,6 +2259,7 @@ class TestDataExchange extends TestBizModel
 					'name' => array(
 						'do_reconcile' => true,
 						'do_update' => true,
+						'automatic_prefix' => true, // unique id
 					),
 					'status' => array(
 						'do_reconcile' => false,