synchro_import.php 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803
  1. <?php
  2. // Copyright (C) 2011-2017 Combodo SARL
  3. //
  4. // This file is part of iTop.
  5. //
  6. // iTop is free software; you can redistribute it and/or modify
  7. // it under the terms of the GNU Affero General Public License as published by
  8. // the Free Software Foundation, either version 3 of the License, or
  9. // (at your option) any later version.
  10. //
  11. // iTop is distributed in the hope that it will be useful,
  12. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. // GNU Affero General Public License for more details.
  15. //
  16. // You should have received a copy of the GNU Affero General Public License
  17. // along with iTop. If not, see <http://www.gnu.org/licenses/>
  18. /**
  19. * Data Exchange web service
  20. *
  21. * @copyright Copyright (C) 2010-2017 Combodo SARL
  22. * @license http://opensource.org/licenses/AGPL-3.0
  23. */
  24. //
  25. // Known limitations
  26. // - reconciliation is made on the column primary_key
  27. //
  28. if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
  29. require_once(__DIR__.'/../approot.inc.php');
  30. require_once(APPROOT.'/application/application.inc.php');
  31. require_once(APPROOT.'/application/webpage.class.inc.php');
  32. require_once(APPROOT.'/application/csvpage.class.inc.php');
  33. require_once(APPROOT.'/application/clipage.class.inc.php');
  34. require_once(APPROOT.'/application/startup.inc.php');
  35. class ExchangeException extends Exception
  36. {
  37. }
  38. $aPageParams = array
  39. (
  40. 'auth_user' => array
  41. (
  42. 'mandatory' => true,
  43. 'modes' => 'cli',
  44. 'default' => null,
  45. 'description' => 'login (must have enough rights to create objects of the given class)',
  46. ),
  47. 'auth_pwd' => array
  48. (
  49. 'mandatory' => true,
  50. 'modes' => 'cli',
  51. 'default' => null,
  52. 'description' => 'password',
  53. ),
  54. 'data_source_id' => array
  55. (
  56. 'mandatory' => true,
  57. 'modes' => 'http,cli',
  58. 'default' => null,
  59. 'description' => 'Synchro data source id',
  60. ),
  61. 'csvdata' => array
  62. (
  63. 'mandatory' => true,
  64. 'modes' => 'http',
  65. 'default' => null,
  66. 'description' => 'data',
  67. ),
  68. 'csvfile' => array
  69. (
  70. 'mandatory' => true,
  71. 'modes' => 'cli',
  72. 'default' => '',
  73. 'description' => 'local data file, replaces csvdata if specified',
  74. ),
  75. 'synchronize' => array
  76. (
  77. 'mandatory' => false,
  78. 'modes' => 'http,cli',
  79. 'default' => '1',
  80. 'description' => 'If set to 1, then the synchronization will be executed right after the data load',
  81. ),
  82. 'charset' => array
  83. (
  84. 'mandatory' => false,
  85. 'modes' => 'http,cli',
  86. 'default' => 'UTF-8',
  87. 'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',
  88. ),
  89. 'date_format' => array
  90. (
  91. 'mandatory' => false,
  92. 'modes' => 'http,cli',
  93. 'default' => 'Y-m-d H:i:s',
  94. 'description' => 'Input date format (used both for dates and datetimes) - Examples: Y-m-d, d/m/Y (Europe) - no transformation is applied if the argument is omitted. (note: old format specification using %Y %m %d is also supported for backward compatibility)',
  95. ),
  96. 'separator' => array
  97. (
  98. 'mandatory' => false,
  99. 'modes' => 'http,cli',
  100. 'default' => ';',
  101. 'description' => 'column separator in CSV data (1 char, or \'tab\')',
  102. ),
  103. 'qualifier' => array
  104. (
  105. 'mandatory' => false,
  106. 'modes' => 'http,cli',
  107. 'default' => '"',
  108. 'description' => 'test qualifier in CSV data',
  109. ),
  110. 'output' => array
  111. (
  112. 'mandatory' => false,
  113. 'modes' => 'http,cli',
  114. 'default' => 'summary',
  115. 'description' => '[retcode] to return the count of lines in error, [summary] to return a concise report, [details] to get a detailed report (each line listed)',
  116. ),
  117. 'max_chunk_size' => array
  118. (
  119. 'mandatory' => false,
  120. 'modes' => 'cli',
  121. 'default' => '0',
  122. 'description' => 'Limit on the count of records that can be loaded at once while performing the synchronization',
  123. ),
  124. /*
  125. 'reportlevel' => array
  126. (
  127. 'mandatory' => false,
  128. 'modes' => 'http,cli',
  129. 'default' => 'errors|warnings|created|changed|unchanged',
  130. 'description' => 'combination of flags to limit the detailed output',
  131. ),
  132. */
  133. 'simulate' => array
  134. (
  135. 'mandatory' => false,
  136. 'modes' => 'http,cli',
  137. 'default' => '0',
  138. 'description' => 'If set to 1, then the load will not be executed, but the expected report will be produced',
  139. ),
  140. 'comment' => array
  141. (
  142. 'mandatory' => false,
  143. 'modes' => 'http,cli',
  144. 'default' => '',
  145. 'description' => 'Comment to be added into the change log',
  146. ),
  147. 'no_stop_on_import_error' => array
  148. (
  149. 'mandatory' => false,
  150. 'modes' => 'http,cli',
  151. 'default' => '0',
  152. 'description' => 'Don\'t stop the import in case of SQL import error. By default the import will stop at the first error (and rollback all changes). If this flag is set to 1 the import will continue anyway',
  153. ),
  154. );
  155. function UsageAndExit($oP)
  156. {
  157. global $aPageParams;
  158. $bModeCLI = utils::IsModeCLI();
  159. $oP->p("USAGE:\n");
  160. foreach($aPageParams as $sParam => $aParamData)
  161. {
  162. $aModes = explode(',', $aParamData['modes']);
  163. if ($bModeCLI)
  164. {
  165. if (in_array('cli', $aModes))
  166. {
  167. $sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
  168. $oP->p("$sParam = $sDesc");
  169. }
  170. }
  171. else
  172. {
  173. if (in_array('http', $aModes))
  174. {
  175. $sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
  176. $oP->p("$sParam = $sDesc");
  177. }
  178. }
  179. }
  180. $oP->output();
  181. exit;
  182. }
  183. function ReadParam($oP, $sParam, $sSanitizationFilter = 'parameter')
  184. {
  185. global $aPageParams;
  186. assert(isset($aPageParams[$sParam]));
  187. assert(!$aPageParams[$sParam]['mandatory']);
  188. $sValue = utils::ReadParam($sParam, $aPageParams[$sParam]['default'], true /* Allow CLI */, $sSanitizationFilter);
  189. return trim($sValue);
  190. }
  191. function ReadMandatoryParam($oP, $sParam, $sSanitizationFilter)
  192. {
  193. global $aPageParams;
  194. assert(isset($aPageParams[$sParam]));
  195. assert($aPageParams[$sParam]['mandatory']);
  196. $sValue = utils::ReadParam($sParam, null, true /* Allow CLI */, $sSanitizationFilter);
  197. if (is_null($sValue))
  198. {
  199. $oP->p("ERROR: Missing argument '$sParam'\n");
  200. UsageAndExit($oP);
  201. }
  202. return trim($sValue);
  203. }
  204. function ChangeDateFormat($sProposedDate, $sFormat, $bDateOnly)
  205. {
  206. if ($sProposedDate == '')
  207. {
  208. // An empty string means 'reset'
  209. return '';
  210. }
  211. // Convert to a valid MySQL datetime
  212. $oFormat = new DateTimeFormat($sFormat);
  213. $sRegExpr = $oFormat->ToRegExpr('/');
  214. if (!preg_match($sRegExpr, $sProposedDate))
  215. {
  216. return false;
  217. }
  218. else
  219. {
  220. $oDate = $oFormat->Parse($sProposedDate);
  221. if ($oDate !== null)
  222. {
  223. $oTargetFormat = $bDateOnly ? AttributeDate::GetInternalFormat() : AttributeDateTime::GetInternalFormat();
  224. $sDate = $oDate->format($oTargetFormat);
  225. return $sDate;
  226. }
  227. else
  228. {
  229. return false;
  230. }
  231. }
  232. }
  233. class CLILikeWebPage extends WebPage
  234. {
  235. public function add_comment($sText)
  236. {
  237. $this->add('#'.$sText."<br/>\n");
  238. }
  239. }
  240. /////////////////////////////////
  241. // Main program
  242. if (utils::IsModeCLI())
  243. {
  244. $oP = new CLIPage(Dict::S("TitleSynchroExecution"));
  245. }
  246. else
  247. {
  248. $oP = new CLILikeWebPage(Dict::S("TitleSynchroExecution"));
  249. }
  250. try
  251. {
  252. utils::UseParamFile();
  253. }
  254. catch(Exception $e)
  255. {
  256. $oP->p("Error: ".$e->GetMessage());
  257. $oP->output();
  258. exit -2;
  259. }
  260. if (utils::IsModeCLI())
  261. {
  262. // Next steps:
  263. // specific arguments: 'csvfile'
  264. //
  265. $sAuthUser = ReadMandatoryParam($oP, 'auth_user', 'raw_data');
  266. $sAuthPwd = ReadMandatoryParam($oP, 'auth_pwd', 'raw_data');
  267. $sCsvFile = ReadMandatoryParam($oP, 'csvfile', 'raw_data');
  268. if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd))
  269. {
  270. UserRights::Login($sAuthUser); // Login & set the user's language
  271. }
  272. else
  273. {
  274. $oP->p("Access restricted or wrong credentials ('$sAuthUser')");
  275. $oP->output();
  276. exit -1;
  277. }
  278. if (!is_readable($sCsvFile))
  279. {
  280. $oP->p("Input file could not be found or could not be read: '$sCsvFile'");
  281. $oP->output();
  282. exit -1;
  283. }
  284. $sCSVData = file_get_contents($sCsvFile);
  285. }
  286. else
  287. {
  288. $_SESSION['login_mode'] = 'basic';
  289. require_once(APPROOT.'/application/loginwebpage.class.inc.php');
  290. LoginWebPage::DoLogin(); // Check user rights and prompt if needed
  291. $sCSVData = utils::ReadPostedParam('csvdata', '', false, 'raw_data');
  292. }
  293. try
  294. {
  295. //////////////////////////////////////////////////
  296. //
  297. // Read parameters
  298. //
  299. $iDataSourceId = ReadMandatoryParam($oP, 'data_source_id', 'raw_data');
  300. $sSynchronize = ReadParam($oP, 'synchronize');
  301. $sSep = ReadParam($oP, 'separator', 'raw_data');
  302. $sQualifier = ReadParam($oP, 'qualifier', 'raw_data');
  303. $sCharSet = ReadParam($oP, 'charset', 'raw_data');
  304. $sDateTimeFormat = ReadParam($oP, 'date_format', 'raw_data');
  305. if ($sDateTimeFormat == '')
  306. {
  307. $sDateTimeFormat = 'Y-m-d H:i:s'; // By default use the SQL date & time format
  308. }
  309. if (strpos($sDateTimeFormat, '%') !== false)
  310. {
  311. $sDateTimeFormat = utils::DateTimeFormatToPHP($sDateTimeFormat);
  312. }
  313. $oDateTimeFormat = new DateTimeFormat($sDateTimeFormat);
  314. $sDateFormat = $oDateTimeFormat->ToDateFormat(); // Keep only the date part
  315. $sOutput = ReadParam($oP, 'output');
  316. // $sReportLevel = ReadParam($oP, 'reportlevel');
  317. $sSimulate = ReadParam($oP, 'simulate');
  318. $sComment = ReadParam($oP, 'comment', 'raw_data');
  319. $sNoStopOnImportError = ReadParam($oP, 'no_stop_on_import_error');
  320. if (strtolower(trim($sSep)) == 'tab')
  321. {
  322. $sSep = "\t";
  323. }
  324. // In case there is a difference between the web server time and the DB server time,
  325. // use the DB server time as a reference since this date/time will be compared with the "status_last_seen"
  326. // column, which is populated by MySQL triggers (and so based on the DB server time)
  327. $oLoadStartDate = new DateTime(CMDBSource::QueryToScalar('SELECT NOW()')); // Now... but as read from the database
  328. // Note about date formatting: These MySQL settings are read-only... and in fact unused :-(
  329. // SET SESSION date_format = '%d/%m/%Y';
  330. // SET SESSION datetime_format = '%d/%m/%Y %H:%i:%s';
  331. // Therefore, we have to allow users to transform the format according to a given specification: date_format
  332. //////////////////////////////////////////////////
  333. //
  334. // Statistics
  335. //
  336. $iCountErrors = 0;
  337. $iCountCreations = 0;
  338. $iCountUpdates = 0;
  339. //////////////////////////////////////////////////
  340. //
  341. // Check parameters format/consistency
  342. //
  343. if (strlen($sCSVData) == 0)
  344. {
  345. throw new ExchangeException("Missing data - at least one line is expected");
  346. }
  347. $oDataSource = MetaModel::GetObject('SynchroDataSource', $iDataSourceId, false);
  348. if (is_null($oDataSource))
  349. {
  350. throw new ExchangeException("Unknown data source id: '$iDataSourceId'");
  351. }
  352. $sClass = $oDataSource->GetTargetClass();
  353. if (strlen($sSep) > 1)
  354. {
  355. throw new ExchangeException("Separator is limited to one character, found '$sSep'");
  356. }
  357. if (strlen($sQualifier) > 1)
  358. {
  359. throw new ExchangeException("Text qualifier is limited to one character, found '$sQualifier'");
  360. }
  361. if (!in_array($sOutput, array('retcode', 'summary', 'details')))
  362. {
  363. throw new ExchangeException("Unknown output format: '$sOutput'");
  364. }
  365. /*
  366. $aReportLevels = explode('|', $sReportLevel);
  367. foreach($aReportLevels as $sLevel)
  368. {
  369. if (!in_array($sLevel, explode('|', 'errors|warnings|created|changed|unchanged')))
  370. {
  371. throw new ExchangeException("Unknown level in reporting level: '$sLevel'");
  372. }
  373. }
  374. */
  375. if ($sSimulate == '1')
  376. {
  377. $bSimulate = true;
  378. }
  379. else
  380. {
  381. $bSimulate = false;
  382. }
  383. if ($sSynchronize == '1')
  384. {
  385. $bSynchronize = true;
  386. }
  387. else
  388. {
  389. $bSynchronize = false;
  390. }
  391. //////////////////////////////////////////////////
  392. //
  393. // Parse first line, check attributes, analyse the request
  394. //
  395. if ($sCharSet == 'UTF-8')
  396. {
  397. $sUTF8Data = $sCSVData;
  398. }
  399. else
  400. {
  401. $sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
  402. }
  403. $oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier, MetaModel::GetConfig()->Get('max_execution_time_per_loop'));
  404. $aInputColumns = $oCSVParser->ListFields();
  405. $iColCount = count($aInputColumns);
  406. // Check columns
  407. $aColumns = $oDataSource->GetSQLColumns();
  408. $aDateColumns = array();
  409. foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
  410. {
  411. if ($oAttDef instanceof AttributeDate)
  412. {
  413. $aDateColumns[$sAttCode] = 'DATE';
  414. }
  415. elseif ($oAttDef instanceof AttributeDateTime)
  416. {
  417. $aDateColumns[$sAttCode] = 'DATETIME';
  418. }
  419. }
  420. $aIsDateToTransform = array();
  421. $aDateToTransformReport = array();
  422. foreach($aInputColumns as $iFieldId => $sInputColumn)
  423. {
  424. if (array_key_exists($sInputColumn, $aDateColumns))
  425. {
  426. $aIsDateToTransform[$iFieldId] = $aDateColumns[$sInputColumn]; // either DATE or DATETIME
  427. $aDateToTransformReport[] = $sInputColumn;
  428. }
  429. else
  430. {
  431. $aIsDateToTransform[$iFieldId] = false;
  432. }
  433. if ($sInputColumn == 'primary_key')
  434. {
  435. $iPrimaryKeyCol = $iFieldId;
  436. continue;
  437. }
  438. if (!array_key_exists($sInputColumn, $aColumns))
  439. {
  440. throw new ExchangeException("Unknown column '$sInputColumn' (class: '$sClass')");
  441. }
  442. }
  443. if (!isset($iPrimaryKeyCol))
  444. {
  445. throw new ExchangeException("Missing reconciliation column 'primary_key'");
  446. }
  447. //////////////////////////////////////////////////
  448. //
  449. // Go for parsing and interpretation
  450. //
  451. try
  452. {
  453. if ($sOutput == 'details')
  454. {
  455. $oP->add_comment('------------------------------------------------------------');
  456. $oP->add_comment(' Import phase');
  457. $oP->add_comment('------------------------------------------------------------');
  458. }
  459. if ($bSimulate)
  460. {
  461. CMDBSource::Query('START TRANSACTION');
  462. }
  463. $aData = $oCSVParser->ToArray();
  464. $iLineCount = count($aData);
  465. $sTable = $oDataSource->GetDataTable();
  466. // Prepare insert columns
  467. $sInsertColumns = '`'.implode('`, `', $aInputColumns).'`';
  468. $iPreviousTimeLimit = ini_get('max_execution_time');
  469. $iLoopTimeLimit = MetaModel::GetConfig()->Get('max_execution_time_per_loop');
  470. $oMutex = new iTopMutex('synchro_import_'.$oDataSource->GetKey());
  471. $oMutex->Lock();
  472. foreach($aData as $iRow => $aRow)
  473. {
  474. set_time_limit($iLoopTimeLimit);
  475. $sReconciliationCondition = "`primary_key` = ".CMDBSource::Quote($aRow[$iPrimaryKeyCol]);
  476. $sSelect = "SELECT COUNT(*) FROM `$sTable` WHERE $sReconciliationCondition";
  477. $aRes = CMDBSource::QueryToArray($sSelect);
  478. $iCount = $aRes[0]['COUNT(*)'];
  479. if ($iCount == 0)
  480. {
  481. // No record... create it
  482. //
  483. $iCountCreations++;
  484. if ($sOutput == 'details')
  485. {
  486. $oP->add("$iRow: New entry, reconciliation: '$sReconciliationCondition'\n");
  487. }
  488. $aValues = array(); // Used to build the insert query
  489. foreach ($aRow as $iCol => $value)
  490. {
  491. if (is_null($value))
  492. {
  493. $aValues[] = 'NULL';
  494. }
  495. elseif ($aIsDateToTransform[$iCol] !== false)
  496. {
  497. $bDateOnly = false;
  498. $sFormat = $sDateTimeFormat;
  499. if ($aIsDateToTransform[$iCol] == 'DATE')
  500. {
  501. $bDateOnly = true;
  502. $sFormat = $sDateFormat;
  503. }
  504. $sDate = ChangeDateFormat($value, $sFormat, $bDateOnly);
  505. if ($sDate === false)
  506. {
  507. $aValues[] = CMDBSource::Quote('');
  508. if ($sOutput == 'details')
  509. {
  510. $oP->add("$iRow: Wrong format for {$aIsDateToTransform[$iCol]} column $iCol: '$value' does not match the expected format: '$sFormat' (column skipped)\n");
  511. }
  512. }
  513. else
  514. {
  515. $aValues[] = CMDBSource::Quote($sDate);
  516. }
  517. }
  518. else
  519. {
  520. $aValues[] = CMDBSource::Quote($value);
  521. }
  522. }
  523. $sValues = implode(', ', $aValues);
  524. $sInsert = "INSERT INTO `$sTable` ($sInsertColumns) VALUES ($sValues)";
  525. try
  526. {
  527. CMDBSource::Query($sInsert);
  528. }
  529. catch(Exception $e)
  530. {
  531. if ($sNoStopOnImportError == '1')
  532. {
  533. $iCountErrors++;
  534. $oP->add("$iRow: Import error '".$e->getMessage()."' (continuing)...\n");
  535. }
  536. else // Fatal error
  537. {
  538. throw $e;
  539. }
  540. }
  541. }
  542. elseif ($iCount == 1)
  543. {
  544. // Found a match... update it
  545. //
  546. $iCountUpdates++;
  547. if ($sOutput == 'details')
  548. {
  549. $oP->add("$iRow: Update entry, reconciliation: '$sReconciliationCondition'\n");
  550. }
  551. $aValuePairs = array(); // Used to build the update query
  552. foreach ($aRow as $iCol => $value)
  553. {
  554. // Skip reconciliation column
  555. if ($iCol == $iPrimaryKeyCol) continue;
  556. $sCol = $aInputColumns[$iCol];
  557. if ($aIsDateToTransform[$iCol] !== false)
  558. {
  559. $bDateOnly = false;
  560. $sFormat = $sDateTimeFormat;
  561. if ($aIsDateToTransform[$iCol] == 'DATE')
  562. {
  563. $bDateOnly = true;
  564. $sFormat = $sDateFormat;
  565. }
  566. $sDate = ChangeDateFormat($value, $sFormat, $bDateOnly);
  567. if ($sDate === false)
  568. {
  569. if ($sOutput == 'details')
  570. {
  571. $oP->add("$iRow: Wrong format for {$aIsDateToTransform[$iCol]} column $iCol: '$value' does not match the expected format: '$sFormat' (column skipped)\n");
  572. }
  573. }
  574. else
  575. {
  576. $aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($sDate);
  577. }
  578. }
  579. else
  580. {
  581. $aValuePairs[] = "`$sCol` = ".CMDBSource::Quote($aRow[$iCol]);
  582. }
  583. }
  584. $sValuePairs = implode(', ', $aValuePairs);
  585. $sUpdateQuery = "UPDATE `$sTable` SET $sValuePairs WHERE $sReconciliationCondition";
  586. try
  587. {
  588. CMDBSource::Query($sUpdateQuery);
  589. }
  590. catch(Exception $e)
  591. {
  592. if ($sNoStopOnImportError == '1')
  593. {
  594. $iCountErrors++;
  595. $oP->add("$iRow: Import error '".$e->getMessage()."' (continuing)...\n");
  596. }
  597. else // Fatal error
  598. {
  599. throw $e;
  600. }
  601. }
  602. }
  603. else
  604. {
  605. // Too many records... ambiguity
  606. //
  607. $iCountErrors++;
  608. $oP->add("$iRow: Error - Failed to reconcile, found $iCount rows having '$sReconciliationCondition'\n");
  609. }
  610. }
  611. $oMutex->Unlock();
  612. set_time_limit($iPreviousTimeLimit);
  613. if (($sOutput == "summary") || ($sOutput == 'details'))
  614. {
  615. $oP->add_comment('------------------------------------------------------------');
  616. $oP->add_comment(' Import phase summary');
  617. $oP->add_comment('------------------------------------------------------------');
  618. $oP->add_comment("Data Source: ".$iDataSourceId);
  619. $oP->add_comment("Synchronize: ".($bSynchronize ? '1' : '0'));
  620. $oP->add_comment("Class: ".$sClass);
  621. $oP->add_comment("Separator: ".$sSep);
  622. $oP->add_comment("Qualifier: ".$sQualifier);
  623. $oP->add_comment("Charset Encoding:".$sCharSet);
  624. if (strlen($sDateTimeFormat) > 0)
  625. {
  626. $aDateTimeColumns = array();
  627. $aDateColumns = array();
  628. foreach($aIsDateToTransform as $iCol => $sType)
  629. {
  630. if ($sType !== false)
  631. {
  632. $sCol = $aInputColumns[$iCol];
  633. if ($sType == 'DATE')
  634. {
  635. $aDateColumns[] = $sCol;
  636. }
  637. else
  638. {
  639. $aDateTimeColumns[] = $sCol;
  640. }
  641. }
  642. }
  643. $sFormatedDateTimeColumns = (count($aDateTimeColumns) > 0) ? ', applied to columns ['.implode(', ', $aDateTimeColumns).']' : '';
  644. $sFormatedDateColumns = (count($aDateColumns) > 0) ? ', applied to columns ['.implode(', ', $aDateColumns).']' : '';
  645. $oP->add_comment("Date and time format: '$sDateTimeFormat' $sFormatedDateTimeColumns");
  646. $oP->add_comment("Date only format: '$sDateFormat' $sFormatedDateColumns");
  647. }
  648. else
  649. {
  650. // shall never get there
  651. $oP->add_comment("Date format: <none>");
  652. }
  653. $oP->add_comment("Data Size: ".strlen($sCSVData));
  654. $oP->add_comment("Data Lines: ".$iLineCount);
  655. $oP->add_comment("Columns: ".implode(', ', $aInputColumns));
  656. $oP->add_comment("Output format: ".$sOutput);
  657. // $oP->add_comment("Report level: ".$sReportLevel);
  658. $oP->add_comment("Simulate: ".($bSimulate ? '1' : '0'));
  659. $oP->add_comment("Change tracking comment: ".$sComment);
  660. $oP->add_comment("Issues (before synchro): ".$iCountErrors);
  661. // $oP->add_comment("Warnings: ".$iCountWarnings);
  662. $oP->add_comment("Created (before synchro): ".$iCountCreations);
  663. $oP->add_comment("Updated (before synchro): ".$iCountUpdates);
  664. }
  665. //////////////////////////////////////////////////
  666. //
  667. // Synchronize
  668. //
  669. if ($bSynchronize)
  670. {
  671. $oSynchroExec = new SynchroExecution($oDataSource, $oLoadStartDate);
  672. $oStatLog = $oSynchroExec->Process();
  673. if ($sOutput == 'details')
  674. {
  675. $oP->add_comment('------------------------------------------------------------');
  676. $oP->add_comment(' Synchronization phase');
  677. $oP->add_comment('------------------------------------------------------------');
  678. $iCount = 0;
  679. foreach ($oStatLog->GetTraces() as $sMessage)
  680. {
  681. $iCount++;
  682. $oP->add_comment($sMessage);
  683. }
  684. if ($iCount == 0)
  685. {
  686. $oP->add_comment(' No traces. (To activate the traces set "synchro_trace" => true in the configuration file)');
  687. }
  688. }
  689. if ($oStatLog->Get('status') == 'error')
  690. {
  691. $oP->p("ERROR: ".$oStatLog->Get('last_error'));
  692. }
  693. $oP->add_comment('------------------------------------------------------------');
  694. $oP->add_comment(' Synchronization phase summary');
  695. $oP->add_comment('------------------------------------------------------------');
  696. $oP->add_comment("Replicas: ".$oStatLog->Get('stats_nb_replica_total'));
  697. $oP->add_comment("Replicas touched since last synchro: ".$oStatLog->Get('stats_nb_replica_seen'));
  698. $oP->add_comment("Objects deleted: ".$oStatLog->Get('stats_nb_obj_deleted'));
  699. $oP->add_comment("Objects deletion errors: ".$oStatLog->Get('stats_nb_obj_deleted_errors'));
  700. $oP->add_comment("Objects obsoleted: ".$oStatLog->Get('stats_nb_obj_obsoleted'));
  701. $oP->add_comment("Objects obsolescence errors: ".$oStatLog->Get('stats_nb_obj_obsoleted_errors'));
  702. $oP->add_comment("Objects created: ".$oStatLog->Get('stats_nb_obj_created')." (".$oStatLog->Get('stats_nb_obj_created_warnings')." warnings)");
  703. $oP->add_comment("Objects creation errors: ".$oStatLog->Get('stats_nb_obj_created_errors'));
  704. $oP->add_comment("Objects updated: ".$oStatLog->Get('stats_nb_obj_updated')." (".$oStatLog->Get('stats_nb_obj_updated_warnings')." warnings)");
  705. $oP->add_comment("Objects update errors: ".$oStatLog->Get('stats_nb_obj_updated_errors'));
  706. $oP->add_comment("Objects reconciled (updated): ".$oStatLog->Get('stats_nb_obj_new_updated')." (".$oStatLog->Get('stats_nb_obj_new_updated_warnings')." warnings)");
  707. $oP->add_comment("Objects reconciled (unchanged): ".$oStatLog->Get('stats_nb_obj_new_unchanged')." (".$oStatLog->Get('stats_nb_obj_new_updated_warnings')." warnings)");
  708. $oP->add_comment("Objects reconciliation errors: ".$oStatLog->Get('stats_nb_replica_reconciled_errors'));
  709. $oP->add_comment("Replica disappeared, no action taken: ".$oStatLog->Get('stats_nb_replica_disappeared_no_action'));
  710. }
  711. }
  712. catch(Exception $e)
  713. {
  714. if ($bSimulate)
  715. {
  716. CMDBSource::Query('ROLLBACK');
  717. }
  718. throw $e;
  719. }
  720. if ($bSimulate)
  721. {
  722. CMDBSource::Query('ROLLBACK');
  723. }
  724. //////////////////////////////////////////////////
  725. //
  726. // Summary of settings and results
  727. //
  728. if ($sOutput == 'retcode')
  729. {
  730. $oP->add($iCountErrors);
  731. }
  732. }
  733. catch(ExchangeException $e)
  734. {
  735. $oP->add_comment($e->getMessage());
  736. }
  737. catch(SecurityException $e)
  738. {
  739. $oP->add_comment($e->getMessage());
  740. }
  741. catch(Exception $e)
  742. {
  743. $oP->add_comment((string)$e);
  744. }
  745. $oP->output();
  746. ?>