Home Explore Help
Register Sign In
SKZH
/
ITSSTOOLS
3
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Browse Source

Improved import.php and synchro_import.php: added 'date_format' (example: %d/%m/%Y %H:%i:%s)

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@1272 a333f486-631f-4898-b8df-5754b55c2be0
romainq 14 years ago
parent
f0b144739a
commit
0a4cc0e1f6
5 changed files with 253 additions and 39 deletions
Unified View Show Diff Stats
  1. 63 16
      application/utils.inc.php
  2. 60 21
      core/bulkchange.class.inc.php
  3. 87 1
      synchro/synchro_import.php
  4. 18 0
      synchro/synchrodatasource.class.inc.php
  5. 25 1
      webservices/import.php

+ 63 - 16
application/utils.inc.php
View File 

@@ -264,12 +264,59 @@ class utils
 
         return $iReturn;

         return $iReturn;

     }

     }

 

 

-    /**

-     * Returns an absolute URL to the current page

-     * @param $bQueryString bool True to also get the query string, false otherwise

-     * @param $bForceHTTPS bool True to force HTTPS, false otherwise

-     * @return string The absolute URL to the current page

-     */                   

+	/**

+	 * Helper function to convert a string to a date, given a format specification. It replaces strtotime which does not allow for specifying a date in a french format (for instance)

+	 * Example: StringToTime('01/05/11 12:03:45', '%d/%m/%y %H:%i:%s')

+	 * @param string $sDate

+	 * @param string $sFormat

+	 * @return timestamp or false if the input format is not correct

+	 */	 	  

+	public static function StringToTime($sDate, $sFormat)

+	{

+	   // Source: http://php.net/manual/fr/function.strftime.php

+		// (alternative: http://www.php.net/manual/fr/datetime.formats.date.php)

+		static $aDateTokens = null;

+		static $aDateRegexps = null;

+		if (is_null($aDateTokens))

+		{

+		   $aSpec = array(

+				'%d' =>'(?<day>[0-9]{2})',

+				'%m' => '(?<month>[0-9]{2})',

+				'%y' => '(?<year>[0-9]{2})',

+				'%Y' => '(?<year>[0-9]{4})',

+				'%H' => '(?<hour>[0-2][0-9])',

+				'%i' => '(?<minute>[0-5][0-9])',

+				'%s' => '(?<second>[0-5][0-9])',

+				);

+			$aDateTokens = array_keys($aSpec);

+			$aDateRegexps = array_values($aSpec);

+		}

+	   

+	   $sDateRegexp = str_replace($aDateTokens, $aDateRegexps, $sFormat);

+	   

+	   if (preg_match('!^(?<head>)'.$sDateRegexp.'(?<tail>)$!', $sDate, $aMatches))

+	   {

+			$sYear = isset($aMatches['year']) ? $aMatches['year'] : 0;

+			$sMonth = isset($aMatches['month']) ? $aMatches['month'] : 1;

+			$sDay = isset($aMatches['day']) ? $aMatches['day'] : 1;

+			$sHour = isset($aMatches['hour']) ? $aMatches['hour'] : 0;

+			$sMinute = isset($aMatches['minute']) ? $aMatches['minute'] : 0;

+			$sSecond = isset($aMatches['second']) ? $aMatches['second'] : 0;

+			return strtotime("$sYear-$sMonth-$sDay $sHour:$sMinute:$sSecond");

+		}

+	   else

+	   {

+	   	return false;

+	   }

+	   // http://www.spaweditor.com/scripts/regex/index.php

+	}

+

+	/**

+	 * Returns an absolute URL to the current page

+	 * @param $bQueryString bool True to also get the query string, false otherwise

+	 * @param $bForceHTTPS bool True to force HTTPS, false otherwise

+	 * @return string The absolute URL to the current page

+	 */                   

 	static public function GetAbsoluteUrl($bQueryString = true, $bForceHTTPS = false)

 	static public function GetAbsoluteUrl($bQueryString = true, $bForceHTTPS = false)

 	{

 	{

 		// Build an absolute URL to this page on this server/port

 		// Build an absolute URL to this page on this server/port

@@ -313,16 +360,16 @@ class utils
 
 			}

 			}

 			$_SERVER['REQUEST_URI'] = $sPath;

 			$_SERVER['REQUEST_URI'] = $sPath;

 		}

 		}

-   		$sPath = $_SERVER['REQUEST_URI'];

-        if (!$bQueryString)

-        {

-            // remove all the parameters from the query string

-            $iQuestionMarkPos = strpos($sPath, '?');

-            if ($iQuestionMarkPos !== false)

-            {

-                $sPath = substr($sPath, 0, $iQuestionMarkPos);

-            }

-        } 

+		$sPath = $_SERVER['REQUEST_URI'];

+		if (!$bQueryString)

+		{

+			// remove all the parameters from the query string

+			$iQuestionMarkPos = strpos($sPath, '?');

+			if ($iQuestionMarkPos !== false)

+			{

+				$sPath = substr($sPath, 0, $iQuestionMarkPos);

+			}

+		} 

 		$sUrl = "$sProtocol://{$sServerName}{$sPort}{$sPath}";

 		$sUrl = "$sProtocol://{$sServerName}{$sPort}{$sPath}";

 		

 		

 		return $sUrl;

 		return $sUrl;

+ 60 - 21
core/bulkchange.class.inc.php
View File 

@@ -252,8 +252,9 @@ class BulkChange
 
 	protected $m_aReconcilKeys; // attcode (attcode = 'id' for the pkey)

 	protected $m_aReconcilKeys; // attcode (attcode = 'id' for the pkey)

 	protected $m_sSynchroScope; // OQL - if specified, then the missing items will be reported

 	protected $m_sSynchroScope; // OQL - if specified, then the missing items will be reported

 	protected $m_aOnDisappear; // array of attcode => value, values to be set when an object gets out of scope (ignored if no scope has been defined)

 	protected $m_aOnDisappear; // array of attcode => value, values to be set when an object gets out of scope (ignored if no scope has been defined)

+	protected $m_sDateFormat; // Date format specification, see utils::StringToTime()

 

 

-	public function __construct($sClass, $aData, $aAttList, $aExtKeys, $aReconcilKeys, $sSynchroScope = null, $aOnDisappear = null)

+	public function __construct($sClass, $aData, $aAttList, $aExtKeys, $aReconcilKeys, $sSynchroScope = null, $aOnDisappear = null, $sDateFormat = null)

 	{

 	{

 		$this->m_sClass = $sClass;

 		$this->m_sClass = $sClass;

 		$this->m_aData = $aData;

 		$this->m_aData = $aData;

@@ -262,6 +263,7 @@ class BulkChange
 
 		$this->m_aExtKeys = $aExtKeys;

 		$this->m_aExtKeys = $aExtKeys;

 		$this->m_sSynchroScope = $sSynchroScope;

 		$this->m_sSynchroScope = $sSynchroScope;

 		$this->m_aOnDisappear = $aOnDisappear;

 		$this->m_aOnDisappear = $aOnDisappear;

+		$this->m_sDateFormat = $sDateFormat;

 	}

 	}

 

 

 	protected $m_bReportHtml = false;

 	protected $m_bReportHtml = false;

@@ -412,7 +414,7 @@ class BulkChange
 
 				}

 				}

 			}

 			}

 			else

 			else

-			{ 

+			{

 				$res = $oTargetObj->CheckValue($sAttCode, $aRowData[$iCol]);

 				$res = $oTargetObj->CheckValue($sAttCode, $aRowData[$iCol]);

 				if ($res === true)

 				if ($res === true)

 				{

 				{

@@ -702,6 +704,35 @@ class BulkChange
 
 			exit;

 			exit;

 		}

 		}

 

 

+		$aResult = array();

+

+		if (!is_null($this->m_sDateFormat) && (strlen($this->m_sDateFormat) > 0))

+		{

+			// Translate dates from the source data

+			//

+			foreach ($this->m_aAttList as $sAttCode => $iCol)

+			{

+				$oAttDef = MetaModel::GetAttributeDef($this->m_sClass, $sAttCode);

+				if ($oAttDef instanceof AttributeDateTime)

+				{

+					foreach($this->m_aData as $iRow => $aRowData)

+					{

+						$sNewDate = utils::StringToTime($this->m_aData[$iRow][$iCol], $this->m_sDateFormat);

+						if ($sNewDate !== false)

+						{

+							// Todo - improve the reporting

+							$this->m_aData[$iRow][$iCol] = $sNewDate;

+						}

+						else

+						{

+							// Leave the cell unchanged

+							$aResult[$iRow]["__STATUS__"]= new RowStatus_Issue("wrong date format");

+							$aResult[$iRow][$sAttCode] = new CellStatus_Issue(null, $this->m_aData[$iRow][$iCol], 'Wrong date format');

+						}

+					}

+				}

+			}

+		}

 

 

 		// Compute the results

 		// Compute the results

 		//

 		//

@@ -709,9 +740,13 @@ class BulkChange
 
 		{

 		{

 			$aVisited = array();

 			$aVisited = array();

 		}

 		}

-		$aResult = array();

 		foreach($this->m_aData as $iRow => $aRowData)

 		foreach($this->m_aData as $iRow => $aRowData)

 		{

 		{

+			if (isset($aResult[$iRow]["__STATUS__"]))

+			{

+				// An issue at the earlier steps - skip the rest

+				continue;

+			}

 			$oReconciliationFilter = new CMDBSearchFilter($this->m_sClass);

 			$oReconciliationFilter = new CMDBSearchFilter($this->m_sClass);

 			$bSkipQuery = false;

 			$bSkipQuery = false;

 			foreach($this->m_aReconcilKeys as $sAttCode)

 			foreach($this->m_aReconcilKeys as $sAttCode)

@@ -798,8 +833,28 @@ class BulkChange
 
 					$aResult[$iRow]["finalclass"]= 'n/a';

 					$aResult[$iRow]["finalclass"]= 'n/a';

 				}

 				}

 			}

 			}

-	

-			// Whatever happened, do report the reconciliation values

+		}

+

+		if (!is_null($this->m_sSynchroScope))

+		{

+			// Compute the delta between the scope and visited objects

+			$oScopeSearch = DBObjectSearch::FromOQL($this->m_sSynchroScope);

+			$oScopeSet = new DBObjectSet($oScopeSearch);

+			while ($oObj = $oScopeSet->Fetch())

+			{

+				$iObj = $oObj->GetKey();

+				if (!in_array($iObj, $aVisited))

+				{

+					$iRow++;

+					$this->UpdateMissingObject($aResult, $iRow, $oObj, $oChange);

+				}

+			}

+		}

+

+		// Fill in the blanks - the result matrix is expected to be 100% complete

+		//

+		foreach($this->m_aData as $iRow => $aRowData)

+		{

 			foreach($this->m_aAttList as $iCol)

 			foreach($this->m_aAttList as $iCol)

 			{

 			{

 				if (!array_key_exists($iCol, $aResult[$iRow]))

 				if (!array_key_exists($iCol, $aResult[$iRow]))

@@ -824,22 +879,6 @@ class BulkChange
 
 			}

 			}

 		}

 		}

 

 

-		if (!is_null($this->m_sSynchroScope))

-		{

-			// Compute the delta between the scope and visited objects

-			$oScopeSearch = DBObjectSearch::FromOQL($this->m_sSynchroScope);

-			$oScopeSet = new DBObjectSet($oScopeSearch);

-			while ($oObj = $oScopeSet->Fetch())

-			{

-				$iObj = $oObj->GetKey();

-				if (!in_array($iObj, $aVisited))

-				{

-					$iRow++;

-					$this->UpdateMissingObject($aResult, $iRow, $oObj, $oChange);

-				}

-			}

-		}

-

 		return $aResult;

 		return $aResult;

 	}

 	}

+ 87 - 1
synchro/synchro_import.php
View File 

@@ -91,6 +91,13 @@ $aPageParams = array
 
 		'default' => 'UTF-8',

 		'default' => 'UTF-8',

 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',

 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',

 	),

 	),

+	'date_format' => array

+	(

+		'mandatory' => false,

+		'modes' => 'http,cli',

+		'default' => '',

+		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',

+	),

 	'separator' => array

 	'separator' => array

 	(

 	(

 		'mandatory' => false,

 		'mandatory' => false,

@@ -192,6 +199,22 @@ function ReadMandatoryParam($oP, $sParam)
 
 	return trim($sValue);

 	return trim($sValue);

 }

 }

 

 

+function ChangeDateFormat($sProposedDate, $sDateFormat)

+{

+	// Make sure this is a valid MySQL datetime

+	$iTime = utils::StringToTime($sProposedDate, $sDateFormat);

+	if ($iTime !== false)

+	{

+		$sDate = date('Y-m-d H:i:s', $iTime);

+		return $sDate;

+	}

+	else

+	{

+		return false;

+	}

+}

+

+

 /////////////////////////////////

 /////////////////////////////////

 // Main program

 // Main program

 

 

@@ -264,6 +287,7 @@ try
 
 	$sSep = ReadParam($oP, 'separator');

 	$sSep = ReadParam($oP, 'separator');

 	$sQualifier = ReadParam($oP, 'qualifier');

 	$sQualifier = ReadParam($oP, 'qualifier');

 	$sCharSet = ReadParam($oP, 'charset');

 	$sCharSet = ReadParam($oP, 'charset');

+	$sDateFormat = ReadParam($oP, 'date_format');

 	$sOutput = ReadParam($oP, 'output');

 	$sOutput = ReadParam($oP, 'output');

 //	$sReportLevel = ReadParam($oP, 'reportlevel');

 //	$sReportLevel = ReadParam($oP, 'reportlevel');

 	$sSimulate = ReadParam($oP, 'simulate');

 	$sSimulate = ReadParam($oP, 'simulate');

@@ -271,6 +295,12 @@ try
 
 

 

 	$oLoadStartDate = new DateTime(); // Now

 	$oLoadStartDate = new DateTime(); // Now

 

 

+   // Note about date formatting: These MySQL settings are read-only... and in fact unused :-(

+	// SET SESSION date_format = '%d/%m/%Y';

+   // SET SESSION datetime_format = '%d/%m/%Y %H:%i:%s';

+   // Therefore, we have to allow users to transform the format according to a given specification: date_format

+

+

 	//////////////////////////////////////////////////

 	//////////////////////////////////////////////////

 	//

 	//

 	// Statistics

 	// Statistics

@@ -358,8 +388,21 @@ try
 
 

 

 	// Check columns

 	// Check columns

 	$aColumns = $oDataSource->GetSQLColumns();

 	$aColumns = $oDataSource->GetSQLColumns();

+	$aDateColumns = $oDataSource->GetDateSQLColumns();

+	$aIsDateToTransform = array();

+	$aDateToTransformReport = array();

 	foreach($aInputColumns as $iFieldId => $sInputColumn)

 	foreach($aInputColumns as $iFieldId => $sInputColumn)

 	{

 	{

+		if ((strlen($sDateFormat) > 0) && (array_key_exists($sInputColumn, $aDateColumns)))

+		{

+			$aIsDateToTransform[$iFieldId] = true;

+			$aDateToTransformReport[] = $sInputColumn;

+		}

+		else

+		{

+			$aIsDateToTransform[$iFieldId] = false;

+		}

+

 		if ($sInputColumn == 'primary_key')

 		if ($sInputColumn == 'primary_key')

 		{

 		{

 			$iPrimaryKeyCol = $iFieldId;

 			$iPrimaryKeyCol = $iFieldId;

@@ -420,6 +463,22 @@ try
 
 					{

 					{

 						$aValues[] = 'NULL';

 						$aValues[] = 'NULL';

 					}

 					}

+					elseif ($aIsDateToTransform[$iCol])

+					{

+						$sDate = ChangeDateFormat($value, $sDateFormat);

+						if ($sDate === false)

+						{

+							$aValues[] = CMDBSource::Quote('');

+							if ($sOutput == 'details')

+							{

+								$oP->add("$iRow: Wrong format for date field: '$value' (skipped column)\n");

+							}

+						}

+						else

+						{

+							$aValues[] = CMDBSource::Quote($sDate);

+						}

+					}

 					else

 					else

 					{

 					{

 						$aValues[] = CMDBSource::Quote($value);

 						$aValues[] = CMDBSource::Quote($value);

@@ -446,7 +505,26 @@ try
 
 					if ($iCol == $iPrimaryKeyCol) continue;

 					if ($iCol == $iPrimaryKeyCol) continue;

 		

 		

 					$sCol = $aInputColumns[$iCol];

 					$sCol = $aInputColumns[$iCol];

-					$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]);

+					if ($aIsDateToTransform[$iCol])

+					{

+						$sDate = ChangeDateFormat($aRow[$iCol], $sDateFormat);

+						if ($sDate === false)

+						{

+							// Skip this column spec

+							if ($sOutput == 'details')

+							{

+								$oP->add("$iRow: Wrong format for date field: '".$aRow[$iCol]."' (skipped column)\n");

+							}

+						}

+						else

+						{

+							$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($sDate);

+						}

+					}

+					else

+					{

+						$aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]);

+					}

 				}

 				}

 				$sValuePairs = implode(', ', $aValuePairs);

 				$sValuePairs = implode(', ', $aValuePairs);

 				$sUpdateQuery = "UPDATE `$sTable` SET $sValuePairs WHERE $sReconciliationCondition";

 				$sUpdateQuery = "UPDATE `$sTable` SET $sValuePairs WHERE $sReconciliationCondition";

@@ -469,6 +547,14 @@ try
 
 			$oP->add_comment("Separator: ".$sSep);

 			$oP->add_comment("Separator: ".$sSep);

 			$oP->add_comment("Qualifier: ".$sQualifier);

 			$oP->add_comment("Qualifier: ".$sQualifier);

 			$oP->add_comment("Charset Encoding:".$sCharSet);

 			$oP->add_comment("Charset Encoding:".$sCharSet);

+			if (strlen($sDateFormat) > 0)

+			{

+				$oP->add_comment("Date format: '$sDateFormat', applied to columns {".implode(', ', $aDateToTransformReport)."}");

+			}

+			else

+			{

+				$oP->add_comment("Date format: <none>");

+			}

 			$oP->add_comment("Data Size: ".strlen($sCSVData));

 			$oP->add_comment("Data Size: ".strlen($sCSVData));

 			$oP->add_comment("Data Lines: ".$iLineCount);

 			$oP->add_comment("Data Lines: ".$iLineCount);

 			$oP->add_comment("Columns: ".implode(', ', $aInputColumns));

 			$oP->add_comment("Columns: ".implode(', ', $aInputColumns));

+ 18 - 0
synchro/synchrodatasource.class.inc.php
View File 

@@ -1095,6 +1095,24 @@ EOF
 
 		return $aColumns;

 		return $aColumns;

 	}

 	}

 	

 	

+	/**

+	 * Get the list of Date and Datetime SQL columns

+	 */

+	public function GetDateSQLColumns()

+	{

+		$aDateAttributes = array();

+		

+		$sClass = $this->GetTargetClass();

+		foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)

+		{

+			if ($oAttDef instanceof AttributeDateTime)

+			{

+				$aDateAttributes[] = $sAttCode;

+			}

+		}

+		return $this->GetSQLColumns($aDateAttributes);

+	}

+

 	public function IsRunning()

 	public function IsRunning()

 	{

 	{

 		$sOQL = "SELECT SynchroLog WHERE sync_source_id = :source_id AND status='running'";

 		$sOQL = "SELECT SynchroLog WHERE sync_source_id = :source_id AND status='running'";

+ 25 - 1
webservices/import.php
View File 

@@ -90,6 +90,13 @@ $aPageParams = array
 
 		'default' => 'UTF-8',

 		'default' => 'UTF-8',

 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',

 		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',

 	),

 	),

+	'date_format' => array

+	(

+		'mandatory' => false,

+		'modes' => 'http,cli',

+		'default' => '',

+		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',

+	),

 	'separator' => array

 	'separator' => array

 	(

 	(

 		'mandatory' => false,

 		'mandatory' => false,

@@ -269,6 +276,7 @@ try
 
 	$sSep = ReadParam($oP, 'separator');

 	$sSep = ReadParam($oP, 'separator');

 	$sQualifier = ReadParam($oP, 'qualifier');

 	$sQualifier = ReadParam($oP, 'qualifier');

 	$sCharSet = ReadParam($oP, 'charset');

 	$sCharSet = ReadParam($oP, 'charset');

+	$sDateFormat = ReadParam($oP, 'date_format');

 	$sOutput = ReadParam($oP, 'output');

 	$sOutput = ReadParam($oP, 'output');

 //	$sReportLevel = ReadParam($oP, 'reportlevel');

 //	$sReportLevel = ReadParam($oP, 'reportlevel');

 	$sReconcKeys = ReadParam($oP, 'reconciliationkeys');

 	$sReconcKeys = ReadParam($oP, 'reconciliationkeys');

@@ -304,6 +312,11 @@ try
 
 		throw new BulkLoadException("Unknown output format: '$sOutput'");

 		throw new BulkLoadException("Unknown output format: '$sOutput'");

 	}

 	}

 

 

+	if (strlen($sDateFormat) == 0)

+	{

+		$sDateFormat = null;

+	}

+	

 /*

 /*

 	$aReportLevels = explode('|', $sReportLevel);

 	$aReportLevels = explode('|', $sReportLevel);

 	foreach($aReportLevels as $sLevel)

 	foreach($aReportLevels as $sLevel)

@@ -331,6 +344,14 @@ try
 
 		$oP->add_comment("Separator: ".$sSep);

 		$oP->add_comment("Separator: ".$sSep);

 		$oP->add_comment("Qualifier: ".$sQualifier);

 		$oP->add_comment("Qualifier: ".$sQualifier);

 		$oP->add_comment("Charset Encoding:".$sCharSet);

 		$oP->add_comment("Charset Encoding:".$sCharSet);

+		if (strlen($sDateFormat) > 0)

+		{

+			$oP->add_comment("Date format: '$sDateFormat'");

+		}

+		else

+		{

+			$oP->add_comment("Date format: <none>");

+		}

 		$oP->add_comment("Data Size: ".strlen($sCSVData));

 		$oP->add_comment("Data Size: ".strlen($sCSVData));

 	}

 	}

 	//////////////////////////////////////////////////

 	//////////////////////////////////////////////////

@@ -592,7 +613,10 @@ try
 
 		$aData,

 		$aData,

 		$aAttList,

 		$aAttList,

 		$aExtKeys,

 		$aExtKeys,

-		$aFinalReconcilKeys

+		$aFinalReconcilKeys,

+		null, // synchro scope

+		null, // on delete

+		$sDateFormat

 	);

 	);

 

 

 	if ($bSimulate)

 	if ($bSimulate)