- Fixed Trac #215: support several characters encoding for the interactive CSV import.

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@726 a333f486-631f-4898-b8df-5754b55c2be0

core/config.class.inc.php

@@ -138,6 +138,12 @@ class Config
 	 */
 	 protected $m_sEncryptionKey;
 
+	/**
+	 * @var array Additional character sets to be supported by the interactive CSV import
+	 *            'iconv_code' => 'display name'
+	 */
+	 protected $m_aCharsets;
+
 	public function __construct($sConfigFile, $bLoadConfig = true)
 	{
 		$this->m_sFile = $sConfigFile;
@@ -192,6 +198,7 @@ class Config
 		$this->m_sAllowedLoginTypes = DEFAULT_ALLOWED_LOGIN_TYPES;
 		$this->m_sExtAuthVariable = DEFAULT_EXT_AUTH_VARIABLE;
 		$this->m_sEncryptionKey = DEFAULT_ENCRYPTION_KEY;
+		$this->m_aCharsets = array();
 		
 		$this->m_aModuleSettings = array();
 
@@ -296,6 +303,7 @@ class Config
 		$this->m_sAllowedLoginTypes = isset($MySettings['allowed_login_types']) ? trim($MySettings['allowed_login_types']) : DEFAULT_ALLOWED_LOGIN_TYPES;
 		$this->m_sExtAuthVariable = isset($MySettings['ext_auth_variable']) ? trim($MySettings['ext_auth_variable']) : DEFAULT_EXT_AUTH_VARIABLE;
 		$this->m_sEncryptionKey = isset($MySettings['encryption_key']) ? trim($MySettings['encryption_key']) : DEFAULT_ENCRYPTION_KEY;
+		$this->m_aCharsets = isset($MySettings['csv_import_charsets']) ? $MySettings['csv_import_charsets'] : array();
 	}
 
 	protected function Verify()
@@ -473,6 +481,11 @@ class Config
 		return $this->m_sExtAuthVariable;
 	}
 
+	public function GetCSVImportCharsets()
+	{
+		return $this->m_aCharsets;
+	}
+	
 	public function SetDBHost($sDBHost)
 	{
 		$this->m_sDBHost = $sDBHost;
@@ -568,6 +581,15 @@ class Config
 		$this->m_sEncryptionKey = $sKey;
 	}
 
+	public function SetCSVImportCharsets($aCharsets)
+	{
+		$this->m_aCharsets = $aCharsets;
+	}
+
+	public function AddCSVImportCharset($sIconvCode, $sDisplayName)
+	{
+		$this->m_aCharsets[$sIconvCode] = $sDisplayName;
+	}	
 	public function FileIsWritable()
 	{
 		return is_writable($this->m_sFile);
@@ -618,6 +640,9 @@ class Config
 			fwrite($hFile, "\t'default_language' => '{$this->m_sDefaultLanguage}',\n");
 			fwrite($hFile, "\t'allowed_login_types' => '{$this->m_sAllowedLoginTypes}',\n");
 			fwrite($hFile, "\t'encryption_key' => '{$this->m_sEncryptionKey}',\n");
+			$sExport = var_export($this->m_aCharsets, true);
+			fwrite($hFile, "\t'csv_import_charsets' => $sExport,\n");
+
 			fwrite($hFile, ");\n");
 
 			fwrite($hFile, "\n");

dictionaries/dictionary.itop.ui.php

@@ -496,7 +496,7 @@ Dict::Add('EN US', 'English', 'English', array(
 	'UI:CSVImport:HeaderSearch' => 'Search?',
 	'UI:CSVImport:AlertIncompleteMapping' => 'Please select a mapping for every field.',
 	'UI:CSVImport:AlertNoSearchCriteria' => 'Please select at least one search criteria',
-	
+	'UI:CSVImport:Encoding' => 'Character encoding',	
 	'UI:UniversalSearchTitle' => 'iTop - Universal Search',
 	'UI:UniversalSearch:Error' => 'Error: %1$s',
 	'UI:UniversalSearch:LabelSelectTheClass' => 'Select the class to search: ',

dictionaries/fr.dictionary.itop.ui.php

@@ -499,6 +499,7 @@ Dict::Add('FR FR', 'French', 'Français', array(
 	'UI:CSVImport:HeaderSearch' => 'Recherche ?',
 	'UI:CSVImport:AlertIncompleteMapping' => 'Veuillez choisir le correspondance de chacun des champs.',
 	'UI:CSVImport:AlertNoSearchCriteria' => 'Veuillez choisir au moins une clef de recherche.',
+	'UI:CSVImport:Encoding' => 'Encodage des caractères',	
 
 	'UI:UniversalSearchTitle' => 'iTop - Recherche Universelle',
 	'UI:UniversalSearch:Error' => 'Erreur : %1$s',

pages/ajax.csvimport.php

@@ -185,6 +185,7 @@ switch($sOperation)
 	$sTextQualifier = utils::ReadParam('qualifier', '"');
 	$iLinesToSkip = utils::ReadParam('nb_lines_skipped', 0);
 	$bFirstLineAsHeader = utils::ReadParam('header_line', true);
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 	$sData = stripslashes(utils::ReadParam('csvdata', true));
 	$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
 	$aData = $oCSVParser->ToArray($iLinesToSkip);
@@ -246,7 +247,7 @@ switch($sOperation)
 	$sData = stripslashes(utils::ReadParam('csvdata', true));
 	$sClassName = utils::ReadParam('class_name', '');
 	$bAdvanced = utils::ReadParam('advanced', false);
-	
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 	$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
 	$aData = $oCSVParser->ToArray($iLinesToSkip);
 	$iTarget = count($aData);

pages/csvimport.php

@@ -232,6 +232,7 @@ function ProcessCSVData(WebPage $oPage, $bSimulate = true)
 	$aSearchFields = utils::ReadParam('search_field', array());
 	$iCurrentStep = $bSimulate ? 4 : 5;
 	$bAdvanced = utils::ReadParam('advanced', 0);
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 	
 	// Parse the data set
 	$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier);
@@ -476,6 +477,7 @@ function ProcessCSVData(WebPage $oPage, $bSimulate = true)
 	$oPage->add('<input type="hidden" name="csvdata_truncated" value="'.htmlentities($sCSVDataTruncated, ENT_QUOTES, 'UTF-8').'"/>');
 	$oPage->add('<input type="hidden" name="class_name" value="'.$sClassName.'"/>');
 	$oPage->add('<input type="hidden" name="advanced" value="'.$bAdvanced.'"/>');
+	$oPage->add('<input type="hidden" name="encoding" value="'.$sEncoding.'"/>');
 	foreach($aFieldsMapping as $iNumber => $sAttCode)
 	{
 		$oPage->add('<input type="hidden" name="field['.$iNumber.']" value="'.$sAttCode.'"/>');
@@ -603,6 +605,7 @@ function SelectMapping(WebPage $oPage)
 	}
 	$sClassName = utils::ReadParam('class_name', '');
 	$bAdvanced = utils::ReadParam('advanced', 0);
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 
 	$oPage->add('<h2>'.Dict::S('UI:Title:CSVImportStep3').'</h2>');
 	$oPage->add('<div class="wizContainer">');
@@ -619,7 +622,7 @@ function SelectMapping(WebPage $oPage)
 	$oPage->add('<input type="hidden" name="nb_skipped_lines" value="'.$iSkippedLines.'"/>');
 	$oPage->add('<input type="hidden" name="csvdata_truncated" id="csvdata_truncated" value="'.htmlentities($sCSVDataTruncated, ENT_QUOTES, 'UTF-8').'"/>');
 	$oPage->add('<input type="hidden" name="csvdata" value="'.htmlentities($sCSVData, ENT_QUOTES, 'UTF-8').'"/>');
-	$oPage->add('<input type="hidden" name="_charset_"/>');
+	$oPage->add('<input type="hidden" name="encoding" value="'.$sEncoding.'">');
 	$oPage->add('<p><input type="button" value="'.Dict::S('UI:Button:Back').'" onClick="CSVGoBack()"/>&nbsp;&nbsp;');
 	$oPage->add('<input type="submit" value="'.Dict::S('UI:Button:SimulateImport').'"/></p>');
 	$oPage->add('</form>');
@@ -670,6 +673,7 @@ EOF
 			var header_line = $('input[name=header_line]').val();
 			var nb_lines_skipped = $('input[name=nb_skipped_lines]').val();
 			var csv_data = $('input[name=csvdata]').val();
+			var encoding = $('input[name=encoding]').val();
 			if (advanced != 1)
 			{
 				advanced = 0;
@@ -687,7 +691,7 @@ EOF
 			ajax_request = $.post('ajax.csvimport.php',
 				   { operation: 'display_mapping_form', enctype: 'multipart/form-data', csvdata: csv_data, separator: separator, 
 				   	 qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line, class_name: class_name,
-				   	 advanced: advanced },
+				   	 advanced: advanced, encoding: encoding },
 				   function(data) {
 					 $('#mapping').empty();
 					 $('#mapping').append(data);
@@ -823,8 +827,19 @@ function SelectOptions(WebPage $oPage)
 		default:
 		$sCSVData = utils::ReadParam('csvdata', '', 'post');
 	}
-	
-	$aGuesses = GuessParameters($sCSVData); // Try to predict the parameters, based on the input data
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
+
+	// Compute a subset of the data set, now that we know the charset
+	if ($sEncoding == 'UTF-8')
+	{
+		$sUTF8Data = $sCSVData;		
+	}
+	else
+	{
+		$sUTF8Data = iconv($sEncoding, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
+	}
+
+	$aGuesses = GuessParameters($sUTF8Data); // Try to predict the parameters, based on the input data
 	
 	$sSeparator = utils::ReadParam('separator', '');
 	if ($sSeparator == '') // May be set to an empty value by the previous page
@@ -848,23 +863,23 @@ function SelectOptions(WebPage $oPage)
 	// Create a truncated version of the data used for the fast preview
 	// Take about 20 lines of data... knowing that some lines may contain carriage returns
 	$iMaxLines = 20;
-	$iMaxLen = strlen($sCSVData);
+	$iMaxLen = strlen($sUTF8Data);
 	$iCurPos = true;
 	while ( ($iCurPos > 0) && ($iMaxLines > 0))
 	{
-		$pos = strpos($sCSVData, "\n", $iCurPos);
+		$pos = strpos($sUTF8Data, "\n", $iCurPos);
 		if ($pos !== false)
 		{
 			$iCurPos = 1+$pos;
 		}
 		else
 		{
-			$iCurPos = strlen($sCSVData);
+			$iCurPos = strlen($sUTF8Data);
 			$iMaxLines = 1;
 		}
 		$iMaxLines--;
 	}
-	$sCSVDataTruncated = substr($sCSVData, 0, $iCurPos);
+	$sCSVDataTruncated = substr($sUTF8Data, 0, $iCurPos);
 	
 	$oPage->add('<h2>'.Dict::S('UI:Title:CSVImportStep2').'</h2>');
 	$oPage->add('<div class="wizContainer">');
@@ -888,7 +903,7 @@ function SelectOptions(WebPage $oPage)
 	$oPage->add('<p><input type="checkbox" name="box_skiplines" value="1" id="box_skiplines" onChange="DoPreview()"'.IsChecked($bBoxSkipLines, 1).'/> '.Dict::Format('UI:CSVImport:Skip_N_LinesAtTheBeginning', '<input type="text" size=2 name="nb_skipped_lines" id="nb_skipped_lines" onChange="DoPreview()" value="'.$iSkippedLines.'">').'<p>');
 	$oPage->add('</td></tr></table>');
 	$oPage->add('<input type="hidden" name="csvdata_truncated" id="csvdata_truncated" value="'.htmlentities($sCSVDataTruncated, ENT_QUOTES, 'UTF-8').'"/>');
-	$oPage->add('<input type="hidden" name="csvdata" id="csvdata" value="'.htmlentities($sCSVData, ENT_QUOTES, 'UTF-8').'"/>');
+	$oPage->add('<input type="hidden" name="csvdata" id="csvdata" value="'.htmlentities($sUTF8Data, ENT_QUOTES, 'UTF-8').'"/>');
 	$oPage->add('<input type="hidden" name="class_name" value="'.$sClassName.'"/>');
 	$oPage->add('<input type="hidden" name="advanced" value="'.$bAdvanced.'"/>');
 	$oPage->add('<input type="hidden" name="step" value="3"/>');
@@ -933,6 +948,7 @@ function SelectOptions(WebPage $oPage)
 		{
 			header_line = 1;
 		}
+		var encoding = $('input[name=encoding]').val();
 
 		$('#preview').block();
 		
@@ -945,7 +961,7 @@ function SelectOptions(WebPage $oPage)
 		}
 		
 		ajax_request = $.post('ajax.csvimport.php',
-			   { operation: 'parser_preview', enctype: 'multipart/form-data', csvdata: $("#csvdata_truncated").val(), separator: separator, qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line },
+			   { operation: 'parser_preview', enctype: 'multipart/form-data', csvdata: $("#csvdata_truncated").val(), separator: separator, qualifier: text_qualifier, nb_lines_skipped: nb_lines_skipped, header_line: header_line, encoding: encoding },
 			   function(data) {
 				 $('#preview').empty();
 				 $('#preview').append(data);
@@ -965,6 +981,23 @@ EOF
  */
 function Welcome(iTopWebPage $oPage)
 {
+	// Encodings supported:
+	// ICONV_CODE => Display Name
+	// Each iconv installation supports different encodings
+	// Some reasonably common and useful encodnings are listed here
+	$aPossibleEncodings = array(
+		'UTF-8' => 'Unicode (UTF-8)',
+		'ISO-8859-1' => 'Western (ISO-8859-1)',
+		'WINDOWS-1251' => 'Cyrilic (Windows 1251)',
+		'WINDOWS-1252' => 'Western (Windows 1252)',
+		'ISO-8859-15' => 'Western (ISO-8859-15)',
+	);
+	// Some more encodings can be specified in the config file
+	$aExtraCharsets = utils::GetConfig()->GetCSVImportCharsets();
+	
+	$aPossibleEncodings = array_merge($aPossibleEncodings, $aExtraCharsets);
+	asort($aPossibleEncodings);
+
 	$oPage->add("<div><p><h1>".Dict::S('UI:Title:BulkImport+')."</h1></p></div>\n");
 	$oPage->AddTabContainer('tabs1');	
 
@@ -974,12 +1007,26 @@ function Welcome(iTopWebPage $oPage)
 	$iSkippedLines = utils::ReadParam('nb_skipped_lines', '');
 	$sClassName = utils::ReadParam('class_name', '');
 	$bAdvanced = utils::ReadParam('advanced', 0);
+	$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 
 	$sFileLoadHtml = '<div><form enctype="multipart/form-data" method="post"><p>'.Dict::S('UI:CSVImport:SelectFile').'</p>'.
-			'<p><input type="file" name="csvdata"/></p>'.
-			'<p><input type="submit" value="'.Dict::S('UI:Button:Next').'"/></p>'.
-			'<p><input type="hidden" name="step" value="2"/></p>'.
-			'<p><input type="hidden" name="operation" value="file_upload"/></p>'.
+			'<p><input type="file" name="csvdata"/></p>';
+			
+	$sFileLoadHtml .= '<p>'.Dict::S('UI:CSVImport:Encoding').': ';
+	$sFileLoadHtml .= '<select name="encoding" onChange="DoPreview()"/>';
+	foreach($aPossibleEncodings as $sIconvCode => $sDisplayName )
+	{
+		$sSelected  = '';
+		if ($sEncoding == $sIconvCode)
+		{
+			$sSelected = ' selected';
+		}
+		$sFileLoadHtml .= '<option value="'.$sIconvCode.'"'.$sSelected.'>'.$sDisplayName.'</option>';
+	}
+	$sFileLoadHtml .= '</select></p>';
+	$sFileLoadHtml .= '<p><input type="submit" value="'.Dict::S('UI:Button:Next').'"/></p>'.
+			'<input type="hidden" name="step" value="2"/>'.
+			'<input type="hidden" name="operation" value="file_upload"/>'.
 			'<input type="hidden" name="header_line" value="'.$bHeaderLine.'"/>'.
 			'<input type="hidden" name="nb_skipped_lines" value="'.$iSkippedLines.'"/>'.
 			'<input type="hidden" name="class_name" value="'.$sClassName.'"/>'.
@@ -989,7 +1036,19 @@ function Welcome(iTopWebPage $oPage)
 	$oPage->AddToTab('tabs1', Dict::S('UI:CSVImport:Tab:LoadFromFile'), $sFileLoadHtml);	
 	$sCSVData = utils::ReadParam('csvdata', '');
 	$sPasteDataHtml = '<div><form enctype="multipart/form-data" method="post"><p>'.Dict::S('UI:CSVImport:PasteData').'</p>'.
-			'<p><textarea cols="100" rows="30" name="csvdata">'.htmlentities($sCSVData, ENT_QUOTES, 'UTF-8').'</textarea></p>'.
+					  '<p><textarea cols="100" rows="30" name="csvdata">'.htmlentities($sCSVData, ENT_QUOTES, 'UTF-8').'</textarea></p>';
+	$sPasteDataHtml .= '<p>'.Dict::S('UI:CSVImport:Encoding').': ';
+	$sPasteDataHtml .= '<select name="encoding" onChange="DoPreview()"/>';
+	foreach($aPossibleEncodings as $sIconvCode => $sDisplayName )
+	{
+		$sSelected  = '';
+		if ($sEncoding == $sIconvCode)
+		{
+			$sSelected = ' selected';
+		}
+		$sPasteDataHtml .= '<option value="'.$sIconvCode.'"'.$sSelected.'>'.$sDisplayName.'</option>';
+	}
+	$sPasteDataHtml .= '</select></p>'.
 			'<p><input type="submit" value="'.Dict::S('UI:Button:Next').'"/></p>'.
 			'<input type="hidden" name="step" value="2"/>'.
 			'<input type="hidden" name="operation" value="csv_data"/>'.