#355 CSV Import (non interactive) now supporting localized column headers, making it possible to import directly data generated by the interactive export. NOTE: to achieve this, the default separator is now the coma (whereas the default separator in XCel sheets is the semicolumn)

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@1122 a333f486-631f-4898-b8df-5754b55c2be0

core/bulkchange.class.inc.php

@@ -1059,6 +1059,49 @@ EOF
 		}
 		$oPage->table($aConfig, $aDetails);
 	}	
+
+	/**
+	 * Get the user friendly name for an 'extended' attribute code i.e 'name', becomes 'Name' and 'org_id->name' becomes 'Organization->Name'
+	 * @param string $sClassName The name of the class
+	 * @param string $sAttCodeEx Either an attribute code or ext_key_name->att_code
+	 * @return string A user friendly format of the string: AttributeName or AttributeName->ExtAttributeName
+	 */
+	public static function GetFriendlyAttCodeName($sClassName, $sAttCodeEx)
+	{
+		$sFriendlyName = '';
+		if (preg_match('/(.+)->(.+)/', $sAttCodeEx, $aMatches) > 0)
+		{
+			$sAttribute = $aMatches[1];
+			$sField = $aMatches[2];
+			$oAttDef = MetaModel::GetAttributeDef($sClassName, $sAttribute);
+			if ($oAttDef->IsExternalKey())
+			{
+				$sTargetClass = $oAttDef->GetTargetClass();
+				$oTargetAttDef = MetaModel::GetAttributeDef($sTargetClass, $sField);
+				$sFriendlyName = $oAttDef->GetLabel().'->'.$oTargetAttDef->GetLabel();
+			}
+			else
+			{
+				 // hum, hum... should never happen, we'd better raise an exception
+				 throw(new Exception(Dict::Format('UI:CSVImport:ErrorExtendedAttCode', $sAttCodeEx, $sAttribute, $sClassName)));
+			}
+	
+		}
+		else
+		{
+			if ($sAttCodeEx == 'id')
+			{
+				$sFriendlyName = Dict::S('UI:CSVImport:idField');
+			}
+			else
+			{
+				$oAttDef = MetaModel::GetAttributeDef($sClassName, $sAttCodeEx);
+				$sFriendlyName = $oAttDef->GetLabel();
+			}
+		}
+		return $sFriendlyName;
+	}
+	
 }
 
 

pages/csvimport.php

@@ -93,48 +93,6 @@ try
 	}
 	
 	/**
-	 * Get the user friendly name for an 'extended' attribute code i.e 'name', becomes 'Name' and 'org_id->name' becomes 'Organization->Name'
-	 * @param string $sClassName The name of the class
-	 * @param string $sAttCodeEx Either an attribute code of ext_key_name->att_code
-	 * @return string A user friendly format of the string: AttributeName or AttributeName->ExtAttributeName
-	 */
-	function GetFriendlyAttCodeName($sClassName, $sAttCodeEx)
-	{
-		$sFriendlyName = '';
-		if (preg_match('/(.+)->(.+)/', $sAttCodeEx, $aMatches) > 0)
-		{
-			$sAttribute = $aMatches[1];
-			$sField = $aMatches[2];
-			$oAttDef = MetaModel::GetAttributeDef($sClassName, $sAttribute);
-			if ($oAttDef->IsExternalKey())
-			{
-				$sTargetClass = $oAttDef->GetTargetClass();
-				$oTargetAttDef = MetaModel::GetAttributeDef($sTargetClass, $sField);
-				$sFriendlyName = $oAttDef->GetLabel().'->'.$oTargetAttDef->GetLabel();
-			}
-			else
-			{
-				 // hum, hum... should never happen, we'd better raise an exception
-				 throw(new Exception(Dict::Format('UI:CSVImport:ErrorExtendedAttCode', $sAttCodeEx, $sAttribute, $sClassName)));
-			}
-	
-		}
-		else
-		{
-			if ($sAttCodeEx == 'id')
-			{
-				$sFriendlyName = Dict::S('UI:CSVImport:idField');
-			}
-			else
-			{
-				$oAttDef = MetaModel::GetAttributeDef($sClassName, $sAttCodeEx);
-				$sFriendlyName = $oAttDef->GetLabel();
-			}
-		}
-		return $sFriendlyName;
-	}
-	
-	/**
 	 * Returns the number of occurences of each char from the set in the specified string
 	 * @param string $sString The input data
 	 * @param array $aSet The set of characters to count
@@ -369,7 +327,7 @@ try
 		{
 			if (!empty($sAttCode) && ($sAttCode != ':none:') && ($sAttCode != 'finalclass'))
 			{
-				$sHtml .= "<th style=\"padding:2px;border-right: 2px #fff solid;\">".GetFriendlyAttCodeName($sClassName, $sAttCode)."</th>";
+				$sHtml .= "<th style=\"padding:2px;border-right: 2px #fff solid;\">".BulkChange::GetFriendlyAttCodeName($sClassName, $sAttCode)."</th>";
 			}
 		}
 		$sHtml .= '<th>Message</th>';

test/testlist.inc.php

@@ -1388,12 +1388,19 @@ class TestImportREST extends TestWebServices
 		return 'Test various options and fonctionality of import.php';
 	}
 
-	protected function DoExecSingleLoad($aLoadSpec)
+	protected function DoExecSingleLoad($aLoadSpec, $iTestId = null)
 	{
 		$sCsvData = $aLoadSpec['csvdata'];
 
 		echo "<div style=\"padding: 10;\">\n";
-		echo "<h3 style=\"background-color: #ddddff; padding: 10;\">{$aLoadSpec['desc']}</h3>\n";
+		if (is_null($iTestId))
+		{
+			echo "<h3 style=\"background-color: #ddddff; padding: 10;\">{$aLoadSpec['desc']}</h3>\n";
+		}
+		else
+		{
+			echo "<h3 style=\"background-color: #ddddff; padding: 10;\"><a href=\"?todo=exec&testid=TestImportREST&subtests=$iTestId\">$iTestId</a> - {$aLoadSpec['desc']}</h3>\n";
+		}
 
 		$aPostData = array('csvdata' => $sCsvData);
 
@@ -1467,16 +1474,6 @@ class TestImportREST extends TestWebServices
 				'csvdata' => "xxx",
 			),
 			array(
-				'desc' => 'Wrong report level',
-				'login' => 'admin',
-				'password' => 'admin',
-				'args' => array(
-					'class' => 'NetworkDevice',
-					'reportlevel' => 'errors|ouarnings|changed',
-				),
-				'csvdata' => "xxx",
-			),
-			array(
 				'desc' => 'Weird format, working anyhow...',
 				'login' => 'admin',
 				'password' => 'admin',
@@ -1499,6 +1496,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Organization',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;code\nWorldCompany;WCY",
@@ -1510,6 +1508,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;org_id;address\nParis;1;Centre de la Franca",
@@ -1521,6 +1520,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Person',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "email;name;first_name;org_id;phone\njohn.foo@starac.com;Foo;John;1;+33(1)23456789",
@@ -1532,6 +1532,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Person',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "email;name;first_name;org_id\nemailPASbon;Foo;John;1",
@@ -1543,6 +1544,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Team',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;org_id;location_name\nSquadra Azzura2;1;Paris",
@@ -1554,17 +1556,44 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Server',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;status;owner_name;location_name;location_id->org_name;os_family;os_version;management_ip;cpu;ram;brand;model;serial_number\nlocalhost.;production;Demo;Grenoble;Demo;Ubuntu 9.10;2.6.31-19-generic-#56-Ubuntu SMP Thu Jan 28 01:26:53 UTC 2010;16.16.230.232;Intel(R) Core(TM)2 Duo CPU     T7100  @ 1.80GHz;2005;Hewlett-Packard;HP Compaq 6510b (GM108UC#ABF);CNU7370BNP",
 			),
 			array(
+				'desc' => 'Load server (column header localized in english)',
+				'login' => 'admin',
+				'password' => 'admin',
+				'args' => array(
+					'class' => 'Server',
+					'output' => 'details',
+					'separator' => ';',
+					'reconciliationkeys' => '',
+				),
+				'csvdata' => "Name;Status;Owner Organization;Location;location_id->org_name;OS Family;OS Version;Management IP;CPU;RAM;Brand;Model;Serial  Number\nlocalhost.;production;Demo;Grenoble;Demo;Ubuntu 9.10;2.6.31-19-generic-#56-Ubuntu SMP Thu Jan 28 01:26:53 UTC 2010;16.16.230.232;Intel(R) Core(TM)2 Duo CPU     T7100  @ 1.80GHz;2005;Hewlett-Packard;HP Compaq 6510b (GM108UC#ABF);CNU7370BNP",
+			),
+			array(
+				'desc' => 'Load server (directly from Export results)',
+				'login' => 'admin',
+				'password' => 'admin',
+				'args' => array(
+					'class' => 'Server',
+					'output' => 'details',
+					'reconciliationkeys' => '',
+				),
+				'csvdata' => 'id,Name,Status,Owner organization,Owner organization->Name,Business criticity,Brand,Model,Serial  Number,Asset Reference,Description,Location,Location->Name,Location details,Management IP,Default Gateway,CPU,RAM,Hard Disk,OS Family,OS Version
+1,"dbserver1.demo.com","production",2,"Demo","medium","HP","DL380","","","ouille
+[[Server:webserver.demo.com]]",1,"Grenoble","","10.1.1.10","255.255.255.0","2","16Gb","120Gb","Linux","Debian (Lenny)"',
+			),
+			array(
 				'desc' => 'Load server - wrong value for status',
 				'login' => 'admin',
 				'password' => 'admin',
 				'args' => array(
 					'class' => 'Server',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;status;owner_name;location_name;location_id->org_name;os_family;os_version;management_ip;cpu;ram;brand;model;serial_number\nlocalhost.;Production;Demo;Grenoble;Demo;Ubuntu 9.10;2.6.31-19-generic-#56-Ubuntu SMP Thu Jan 28 01:26:53 UTC 2010;16.16.230.232;Intel(R) Core(TM)2 Duo CPU     T7100  @ 1.80GHz;2005;Hewlett-Packard;HP Compaq 6510b (GM108UC#ABF);CNU7370BNP",
@@ -1576,6 +1605,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'NetworkInterface',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => '',
 				),
 				'csvdata' => "name;status;org_id;device_name;physical_type;ip_address;ip_mask;mac_address;speed\neth0;implementation;2;localhost.;ethernet;16.16.230.232;255.255.240.0;00:1a:4b:68:e3:97;\nlo;implementation;2;localhost.;ethernet;127.0.0.1;255.0.0.0;;",
@@ -1588,6 +1618,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'NetworkDevice',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => 'org_id->name,name',
 					),
 				'csvdata' => 'name;management_ip;importance;org_id->name;type
@@ -1601,6 +1632,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'NetworkInterface',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => 'device_id->name,name',
 				),
 				'csvdata' => 'device_id->name;org_id->name;name;ip_address;ip_mask;speed;link_type;mac_address;physical_type
@@ -1625,6 +1657,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_id\nParis;2",
 			),
@@ -1635,6 +1668,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_name\nParis;Demo",
 			),
@@ -1645,6 +1679,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_id->code\nParis;DEMO",
 			),
@@ -1655,6 +1690,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'summary',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_id->code\nParis;DEMO",
 			),
@@ -1665,6 +1701,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'retcode',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_id->code\nParis;DEMO",
 			),
@@ -1675,6 +1712,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 					'reconciliationkeys' => 'org_id',
 				),
 				'csvdata' => "org_name;name\nDemo;Paris",
@@ -1686,6 +1724,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "org_name;country\nDemo;France",
 			),
@@ -1696,6 +1735,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org\nParis;2",
 			),
@@ -1706,6 +1746,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org->code\nParis;DEMO",
 			),
@@ -1716,6 +1757,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 				),
 				'csvdata' => "name;org_id->duns\nParis;DEMO",
 			),
@@ -1726,6 +1768,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 					'comment' => 'automated testing'
 				),
 				'csvdata' => "org_name;name;address\nDemo;Le pantheon;Addresse bidon:".((string)microtime(true)),
@@ -1737,6 +1780,7 @@ class TestImportREST extends TestWebServices
 				'args' => array(
 					'class' => 'Location',
 					'output' => 'details',
+					'separator' => ';',
 					'simulate' => '1',
 					'comment' => 'SHOULD NEVER APPEAR IN THE HISTORY'
 				),
@@ -1744,9 +1788,21 @@ class TestImportREST extends TestWebServices
 			),
 		); 
 
-		foreach ($aLoads as $aLoadSpec)
+     	$sSubTests = utils::ReadParam('subtests', null);
+     	if (is_null($sSubTests))
+     	{
+			foreach ($aLoads as $iTestId => $aLoadSpec)
+			{
+				$this->DoExecSingleLoad($aLoadSpec, $iTestId);
+			}
+		}
+		else
 		{
-			$this->DoExecSingleLoad($aLoadSpec);
+			$aSubTests = explode(',', $sSubTests);
+			foreach ($aSubTests as $iTestId)
+			{
+				$this->DoExecSingleLoad($aLoads[$iTestId], $iTestId);
+			}
 		}
 	}
 }

webservices/import.php

@@ -93,7 +93,7 @@ $aPageParams = array
 	(
 		'mandatory' => false,
 		'modes' => 'http,cli',
-		'default' => ';',
+		'default' => ',',
 		'description' => 'column separator in CSV data',
 	),
 	'qualifier' => array
@@ -304,6 +304,15 @@ try
 		$bSimulate = false;
 	}
 
+	if (($sOutput == "summary") || ($sOutput == 'details'))
+	{
+		$oP->add_comment("Output format: ".$sOutput);
+		$oP->add_comment("Class: ".$sClass);
+		$oP->add_comment("Separator: ".$sSep);
+		$oP->add_comment("Qualifier: ".$sQualifier);
+		$oP->add_comment("Charset Encoding:".$sCharSet);
+		$oP->add_comment("Data Size: ".strlen($sCSVData));
+	}
 	//////////////////////////////////////////////////
 	//
 	// Security
@@ -315,6 +324,25 @@ try
 
 	//////////////////////////////////////////////////
 	//
+	// Make translated header reference
+	//
+	$aFriendlyToInternalAttCode = array();
+	foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
+	{
+	  	$aFriendlyToInternalAttCode[strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCode))] = $sAttCode;
+	  	if ($oAttDef->IsExternalKey(EXTKEY_RELATIVE))
+	  	{
+	  		$sRemoteClass = $oAttDef->GetTargetClass();
+			foreach(MetaModel::ListAttributeDefs($sRemoteClass) as $sRemoteAttCode => $oRemoteAttDef)
+		  	{
+		  		$sAttCodeEx = $sAttCode.'->'.$sRemoteAttCode;
+		  		$aFriendlyToInternalAttCode[strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCodeEx))] = $sAttCodeEx;
+		  	}
+		}
+   }
+   
+	//////////////////////////////////////////////////
+	//
 	// Parse first line, check attributes, analyse the request
 	//
 	if ($sCharSet == 'UTF-8')
@@ -331,9 +359,24 @@ try
 	$aRawFieldList = $oCSVParser->ListFields();
 	$iColCount = count($aRawFieldList);
 
+	// Translate into internal names
+	$aFieldList = array();
+	foreach($aRawFieldList as $iFieldId => $sFieldName)
+	{
+		$sFieldName = trim($sFieldName);
+		if (array_key_exists(strtolower($sFieldName), $aFriendlyToInternalAttCode))
+		{
+			$aFieldList[$iFieldId] = $aFriendlyToInternalAttCode[strtolower($sFieldName)];
+		}
+		else
+		{
+			$aFieldList[$iFieldId] = $sFieldName;
+		}
+	}	
+
 	$aAttList = array();
 	$aExtKeys = array();
-	foreach($aRawFieldList as $iFieldId => $sFieldName)
+	foreach($aFieldList as $iFieldId => $sFieldName)
 	{
 		$aMatches = array();
 		if (preg_match('/^(.+)\*$/', $sFieldName, $aMatches))
@@ -363,6 +406,10 @@ try
 			}
 			$aExtKeys[$sExtKeyAttCode][$sRemoteAttCode] = $iFieldId;
 		}
+		elseif ($sFieldName == 'id')
+		{
+			$aAttList[$sFieldName] = $iFieldId;
+		}
 		else
 		{
 			// The column has been specified as "attcode"
@@ -399,7 +446,7 @@ try
 		// The reconciliation attributes not present in the data will be ignored
 		foreach(MetaModel::GetReconcKeys($sClass) as $sReconcKeyAttCode)
 		{
-			if (in_array($sReconcKeyAttCode, $aRawFieldList))
+			if (in_array($sReconcKeyAttCode, $aFieldList))
 			{
 				$aReconcSpec[] = $sReconcKeyAttCode;
 			}
@@ -420,7 +467,7 @@ try
 		$sReconcKey = trim($sReconcKey);
 		if (empty($sReconcKey)) continue; // skip empty spec
 
-		if (!in_array($sReconcKey, $aRawFieldList))
+		if (!in_array($sReconcKey, $aFieldList))
 		{
 			throw new BulkLoadException("Reconciliation keys not found in the input columns '$sReconcKey' (class: '$sClass')");
 		}
@@ -472,6 +519,27 @@ try
 	$aData = $oCSVParser->ToArray();
 	$iLineCount = count($aData);
 
+	if (($sOutput == "summary") || ($sOutput == 'details'))
+	{
+		$oP->add_comment("Data Lines: ".$iLineCount);
+		$oP->add_comment("Simulate: ".($bSimulate ? '1' : '0'));
+		$oP->add_comment("Columns: ".implode(', ', $aFieldList));
+
+		$aReconciliationReport = array();
+		foreach($aReconcilKeysReport as $sKey => $aKeyDetails)
+		{
+			if (count($aKeyDetails) > 0)
+			{
+				$aReconciliationReport[] = $sKey.' ('.implode(',', $aKeyDetails).')';
+			}
+			else
+			{
+				$aReconciliationReport[] = $sKey;
+			}
+		}
+		$oP->add_comment("Reconciliation Keys: ".implode(', ', $aReconciliationReport));
+	}
+
 	$oBulk = new BulkChange(
 		$sClass,
 		$aData,
@@ -569,29 +637,7 @@ try
 
 	if (($sOutput == "summary") || ($sOutput == 'details'))
 	{
-		$aReconciliationReport = array();
-		foreach($aReconcilKeysReport as $sKey => $aKeyDetails)
-		{
-			if (count($aKeyDetails) > 0)
-			{
-				$aReconciliationReport[] = $sKey.' ('.implode(',', $aKeyDetails).')';
-			}
-			else
-			{
-				$aReconciliationReport[] = $sKey;
-			}
-		}
-		$oP->add_comment("Class: ".$sClass);
-		$oP->add_comment("Separator: ".$sSep);
-		$oP->add_comment("Qualifier: ".$sQualifier);
-		$oP->add_comment("Charset Encoding:".$sCharSet);
-		$oP->add_comment("Data Size: ".strlen($sCSVData));
-		$oP->add_comment("Data Lines: ".$iLineCount);
-		$oP->add_comment("Columns: ".implode(', ', $aRawFieldList));
-		$oP->add_comment("Reconciliation Keys: ".implode(', ', $aReconciliationReport));
-		$oP->add_comment("Output format: ".$sOutput);
 //		$oP->add_comment("Report level: ".$sReportLevel);
-		$oP->add_comment("Simulate: ".($bSimulate ? '1' : '0'));
 		$oP->add_comment("Change tracking comment: ".$sComment);
 		$oP->add_comment("Issues: ".$iCountErrors);
 		$oP->add_comment("Warnings: ".$iCountWarnings);
@@ -622,8 +668,17 @@ try
 		}
 		foreach ($aAttList as $sAttCode => $iCol)
 		{
-			$sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel();
-			$aDisplayConfig["$iCol"] = array("label"=>$sAttCode, "description"=>$sLabel);
+			if ($sAttCode == 'id')
+			{
+				$sLabel = Dict::S('UI:CSVImport:idField');
+
+				$aDisplayConfig["$iCol"] = array("label"=>$sAttCode, "description"=>$sLabel);
+			}
+			else
+			{
+				$sLabel = MetaModel::GetAttributeDef($sClass, $sAttCode)->GetLabel();
+				$aDisplayConfig["$iCol"] = array("label"=>$sAttCode, "description"=>$sLabel);
+			}
 		}
 	
 		$aResultDisp = array(); // to be displayed