- Make sure that the CSV Parser has enough time to run on big amount of data.

- Speedup the display of the CSV Import interactive wizard by parsing only the needed lines of the CSV data (in the first steps of the wizard).

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@4223 a333f486-631f-4898-b8df-5754b55c2be0

core/csvparser.class.inc.php

@@ -54,12 +54,14 @@ class CSVParser
 	private $m_sCSVData;
 	private $m_sSep;
 	private $m_sTextQualifier;
+	private $m_iTimeLimitPerRow;
 
-	public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"')
+	public function __construct($sTxt, $sSep = ',', $sTextQualifier = '"', $iTimeLimitPerRow = null)
 	{
 		$this->m_sCSVData = str_replace("\r\n", "\n", $sTxt);
 		$this->m_sSep = $sSep;
 		$this->m_sTextQualifier = $sTextQualifier;
+		$this->m_iTimeLimitPerRow = $iTimeLimitPerRow;
 	}
 
 	protected $m_sCurrCell = '';
@@ -129,6 +131,12 @@ class CSVParser
 			// blank line, skip silently
 		}
 		$this->m_aCurrRow = array();
+		
+		// More time for the next row
+		if ($this->m_iTimeLimitPerRow !== null)
+		{
+			set_time_limit($this->m_iTimeLimitPerRow);
+		}
 	}
 	protected function __AddCellTrimmed($c = null, $aFieldMap = null)
 	{
@@ -181,6 +189,13 @@ class CSVParser
 		$iDataLength = strlen($this->m_sCSVData);
 
 		$iState = stSTARTING;
+		$iTimeLimit = null;
+		if ($this->m_iTimeLimitPerRow !== null)
+		{
+			// Give some time for the first row
+			$iTimeLimit = ini_get('max_execution_time');
+			set_time_limit($this->m_iTimeLimitPerRow);
+		}
 		for($i = 0; $i <= $iDataLength ; $i++)
 		{
 			if ($i == $iDataLength)
@@ -237,6 +252,11 @@ class CSVParser
 			$iLineCount = count($this->m_aDataSet);
 			if (($iMax > 0) && ($iLineCount >= $iMax)) break;
 		}
+		if ($iTimeLimit !== null)
+		{
+			// Restore the previous time limit
+			set_time_limit($iTimeLimit);
+		}
 		return $this->m_aDataSet;
 	}
 

pages/ajax.csvimport.php

@@ -244,8 +244,9 @@ try
 		$bFirstLineAsHeader = utils::ReadParam('header_line', true);
 		$sEncoding = utils::ReadParam('encoding', 'UTF-8');
 		$sData = stripslashes(utils::ReadParam('csvdata', true, false, 'raw_data'));
-		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
-		$aData = $oCSVParser->ToArray($iLinesToSkip);
+		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
+		$iMaxIndex= 10; // Display maximum 10 lines for the preview
+		$aData = $oCSVParser->ToArray($iLinesToSkip, null, $iMaxIndex);
 		$iTarget = count($aData);
 		if ($iTarget == 0)
 		{
@@ -258,7 +259,6 @@ try
 			$oPage->p("<h3>".Dict::S('UI:Title:DataPreview')."</h3>\n");
 			$oPage->p("<div style=\"overflow-y:auto\" class=\"white\">\n");
 			$oPage->add("<table cellspacing=\"0\" style=\"overflow-y:auto\">");
-			$iMaxIndex= 10; // Display maximum 10 lines for the preview
 			$index = 1;
 			foreach($aData as $aRow)
 			{
@@ -318,8 +318,8 @@ try
 		$aInitFieldMapping = empty($sInitFieldMapping) ? array() : json_decode($sInitFieldMapping, true);
 		$aInitSearchField = empty($sInitSearchField) ? array() : json_decode($sInitSearchField, true);
 
-		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier);
-		$aData = $oCSVParser->ToArray($iLinesToSkip);
+		$oCSVParser = new CSVParser($sData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
+		$aData = $oCSVParser->ToArray($iLinesToSkip, null, 3 /* Max: 1 header line + 2 lines of sample data */);
 		$iTarget = count($aData);
 		if ($iTarget == 0)
 		{

pages/csvimport.php

@@ -232,7 +232,7 @@ try
 		}
 				
 		// Parse the data set
-		$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier);
+		$oCSVParser = new CSVParser($sCSVData, $sSeparator, $sTextQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
 		$aData = $oCSVParser->ToArray($iSkippedLines);
 		$iRealSkippedLines = $iSkippedLines;
 		if ($bHeaderLine)

synchro/synchro_import.php

@@ -429,7 +429,7 @@ try
 	{
 		$sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
 	}
-	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); 
+	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop')); 
 
 	$aInputColumns = $oCSVParser->ListFields();
 	$iColCount = count($aInputColumns);