Enhancement: support injection of new modules treated as data.

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@3525 a333f486-631f-4898-b8df-5754b55c2be0

core/config.class.inc.php

@@ -1742,93 +1742,92 @@ class Config
 			$this->SetDBSubname($aParamValues['db_prefix']);
 		}
 		
-		if (!is_null($sModulesDir))
-		{
-			if (isset($aParamValues['selected_modules']))
-			{
-				$aSelectedModules = explode(',', $aParamValues['selected_modules']);
-			}
-			else
-			{
-				$aSelectedModules = null;
-			}
-	
-			// Initialize the arrays below with default values for the application...
-			$oEmptyConfig = new Config('dummy_file', false); // Do NOT load any config file, just set the default values
-			$aAddOns = $oEmptyConfig->GetAddOns();
-			$aAppModules = $oEmptyConfig->GetAppModules();
-			if (file_exists(APPROOT.$sModulesDir.'/core/main.php'))
-			{
-				$aAppModules[] = $sModulesDir.'/core/main.php';
-			}
-			$aDataModels = $oEmptyConfig->GetDataModels();
-			$aWebServiceCategories = $oEmptyConfig->GetWebServiceCategories();
-			$aDictionaries = $oEmptyConfig->GetDictionaries();
-			// Merge the values with the ones provided by the modules
-			// Make sure when don't load the same file twice...
-	
-			$aModules = ModuleDiscovery::GetAvailableModules(array(APPROOT.$sModulesDir));
-			foreach($aModules as $sModuleId => $aModuleInfo)
-			{
-				list($sModuleName, $sModuleVersion) = ModuleDiscovery::GetModuleName($sModuleId);
-				if (is_null($aSelectedModules) || in_array($sModuleName, $aSelectedModules))
-				{
-					if (isset($aModuleInfo['datamodel']))
-					{
-						$aDataModels = array_unique(array_merge($aDataModels, $aModuleInfo['datamodel']));
-					}
-					if (isset($aModuleInfo['webservice']))
-					{
-						$aWebServiceCategories = array_unique(array_merge($aWebServiceCategories, $aModuleInfo['webservice']));
-					}
-					if (isset($aModuleInfo['settings']))
-					{
-						list($sName, $sVersion) = ModuleDiscovery::GetModuleName($sModuleId);
-						foreach($aModuleInfo['settings'] as $sProperty => $value)
-						{
-							if ($bPreserveModuleSettings && isset($this->m_aModuleSettings[$sName][$sProperty]))
-							{
-								// Do nothing keep the original value
-							}
-							else
-							{
-								$this->SetModuleSetting($sName, $sProperty, $value);
-							}
-						}
-					}
-					if (isset($aModuleInfo['installer']))
-					{
-						$sModuleInstallerClass = $aModuleInfo['installer'];
-						if (!class_exists($sModuleInstallerClass))
-						{
-							throw new Exception("Wrong installer class: '$sModuleInstallerClass' is not a PHP class - Module: ".$aModuleInfo['label']);
-						}
-						if (!is_subclass_of($sModuleInstallerClass, 'ModuleInstallerAPI'))
-						{
-							throw new Exception("Wrong installer class: '$sModuleInstallerClass' is not derived from 'ModuleInstallerAPI' - Module: ".$aModuleInfo['label']);
-						}
-						$aCallSpec = array($sModuleInstallerClass, 'BeforeWritingConfig');
-						call_user_func_array($aCallSpec, array($this));
-					}
-				}
-			}
-			$this->SetAddOns($aAddOns);
-			$this->SetAppModules($aAppModules);
-			$this->SetDataModels($aDataModels);
-			$this->SetWebServiceCategories($aWebServiceCategories);
-
-			// Scan dictionaries
-			//
-			if (!is_null($sModulesDir))
-			{
-				foreach(glob(APPROOT.$sModulesDir.'/dictionaries/*.dict.php') as $sFilePath)
-				{
-					$sFile = basename($sFilePath);
-					$aDictionaries[] = $sModulesDir.'/dictionaries/'.$sFile;
-				}
-			}
-			$this->SetDictionaries($aDictionaries);
-		}
+		$this->UpdateIncludes($sModulesDir, $aSelectedModules);
+	}
+
+	/**
+	 * Helper function to rebuild the default configuration and the list of includes from a directory and a list of selected modules
+	 * @param string $sModulesDir The relative path to the directory to scan for modules (typically the 'env-xxx' directory resulting from the compilation)
+	 * @param array $aSelectedModules An array of selected modules' identifiers. If null all modules found will be considered as installed
+	 * @throws Exception
+	 */
+	public function UpdateIncludes($sModulesDir, $aSelectedModules = null)
+	{
+		if (!is_null($sModulesDir))
+		{	
+			// Initialize the arrays below with default values for the application...
+			$oEmptyConfig = new Config('dummy_file', false); // Do NOT load any config file, just set the default values
+			$aAddOns = $oEmptyConfig->GetAddOns();
+			$aAppModules = $oEmptyConfig->GetAppModules();
+			if (file_exists(APPROOT.$sModulesDir.'/core/main.php'))
+			{
+				$aAppModules[] = $sModulesDir.'/core/main.php';
+			}
+			$aDataModels = $oEmptyConfig->GetDataModels();
+			$aWebServiceCategories = $oEmptyConfig->GetWebServiceCategories();
+			$aDictionaries = $oEmptyConfig->GetDictionaries();
+			// Merge the values with the ones provided by the modules
+			// Make sure when don't load the same file twice...
+			
+			$aModules = ModuleDiscovery::GetAvailableModules(array(APPROOT.$sModulesDir));
+			foreach ($aModules as $sModuleId => $aModuleInfo)
+			{
+				list ($sModuleName, $sModuleVersion) = ModuleDiscovery::GetModuleName($sModuleId);
+				if (is_null($aSelectedModules) || in_array($sModuleName, $aSelectedModules))
+				{
+					if (isset($aModuleInfo['datamodel']))
+					{
+						$aDataModels = array_unique(array_merge($aDataModels, $aModuleInfo['datamodel']));
+					}
+					if (isset($aModuleInfo['webservice']))
+					{
+						$aWebServiceCategories = array_unique(array_merge($aWebServiceCategories, $aModuleInfo['webservice']));
+					}
+					if (isset($aModuleInfo['settings']))
+					{
+						list ($sName, $sVersion) = ModuleDiscovery::GetModuleName($sModuleId);
+						foreach ($aModuleInfo['settings'] as $sProperty => $value)
+						{
+							if (isset($this->m_aModuleSettings[$sName][$sProperty]))
+							{
+								// Do nothing keep the original value
+							}
+							else
+							{
+								$this->SetModuleSetting($sName, $sProperty, $value);
+							}
+						}
+					}
+					if (isset($aModuleInfo['installer']))
+					{
+						$sModuleInstallerClass = $aModuleInfo['installer'];
+						if (!class_exists($sModuleInstallerClass))
+						{
+							throw new Exception("Wrong installer class: '$sModuleInstallerClass' is not a PHP class - Module: ".$aModuleInfo['label']);
+						}
+						if (!is_subclass_of($sModuleInstallerClass, 'ModuleInstallerAPI'))
+						{
+							throw new Exception("Wrong installer class: '$sModuleInstallerClass' is not derived from 'ModuleInstallerAPI' - Module: ".$aModuleInfo['label']);
+						}
+						$aCallSpec = array($sModuleInstallerClass,'BeforeWritingConfig');
+						call_user_func_array($aCallSpec, array($this));
+					}
+				}
+			}
+			$this->SetAddOns($aAddOns);
+			$this->SetAppModules($aAppModules);
+			$this->SetDataModels($aDataModels);
+			$this->SetWebServiceCategories($aWebServiceCategories);
+			
+			// Scan dictionaries
+			//
+			foreach (glob(APPROOT.$sModulesDir.'/dictionaries/*.dict.php') as $sFilePath)
+			{
+				$sFile = basename($sFilePath);
+				$aDictionaries[] = $sModulesDir.'/dictionaries/'.$sFile;
+			}
+			$this->SetDictionaries($aDictionaries);
+		}
 	}
 
 	/**

datamodels/2.x/itop-backup/ajax.backup.php

@@ -24,7 +24,7 @@
  */
 
 if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
-require_once(__DIR__.'/../../approot.inc.php');
+if (!defined('APPROOT')) require_once(__DIR__.'/../../approot.inc.php');
 require_once(APPROOT.'/application/application.inc.php');
 require_once(APPROOT.'/application/webpage.class.inc.php');
 require_once(APPROOT.'/application/ajaxwebpage.class.inc.php');
@@ -127,6 +127,10 @@ EOF
 				// Get the file and destroy the token (single usage)
 				$sToken = utils::ReadParam('token', '', false, 'raw_data');
 				$sTokenFile = APPROOT.'/data/restore.'.$sToken.'.tok';
+				if (!is_file($sTokenFile))
+				{
+					throw new Exception("Error: missing token file: '$sTokenFile'");
+				}
 				$sFile = file_get_contents($sTokenFile);
 				unlink($sTokenFile);
 	

datamodels/2.x/itop-backup/backup.php

@@ -15,7 +15,7 @@
 //   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
-require_once(__DIR__.'/../../approot.inc.php');
+if (!defined('APPROOT')) require_once(__DIR__.'/../../approot.inc.php');
 require_once(APPROOT.'application/application.inc.php');
 require_once(APPROOT.'application/webpage.class.inc.php');
 require_once(APPROOT.'application/csvpage.class.inc.php');

datamodels/2.x/itop-backup/dbrestore.class.inc.php

@@ -92,7 +92,7 @@ class DBRestore extends DBBackup
 	{
 		$this->LogInfo("Starting restore of ".basename($sZipFile));
 
-		$oZip = new ZipArchive();
+		$oZip = new ZipArchiveEx();
 		$res = $oZip->open($sZipFile);
 
 		// Load the database
@@ -117,6 +117,15 @@ class DBRestore extends DBBackup
 		{
 			@unlink($sDeltaFile);
 		}
+		if (is_dir(APPROOT.'data/production-modules/'))
+		{
+			SetupUtils::rrmdir(APPROOT.'data/production-modules/');
+		}
+		if ($oZip->locateName('production-modules/') !== false)
+		{
+			$oZip->extractDirTo(APPROOT.'data/', 'production-modules/');
+		}
+
 		$sConfigFile = APPROOT.'conf/'.$sEnvironment.'/config-itop.php';
 		@chmod($sConfigFile, 0770); // Allow overwriting the file
 		$oZip->extractTo(APPROOT.'conf/'.$sEnvironment, 'config-itop.php');

datamodels/2.x/itop-backup/status.php

@@ -25,7 +25,7 @@
  */
 
 if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
-require_once(__DIR__.'/../../approot.inc.php');
+if (!defined('APPROOT')) require_once(__DIR__.'/../../approot.inc.php');
 require_once(APPROOT.'application/application.inc.php');
 require_once(APPROOT.'application/itopwebpage.class.inc.php');
 

pages/ajax.render.php

@@ -1162,6 +1162,11 @@ EOF
 		{
 			$aSearchDirs[] = APPROOT.'extensions';
 		}
+		$sExtraDir = APPROOT.'data/'.$sCurrEnv.'-modules/';
+		if (file_exists($sExtraDir))
+		{
+			$aSearchDirs[] = $sExtraDir;
+		}
 		$aAvailableModules = $oRuntimeEnv->AnalyzeInstallation(MetaModel::GetConfig(), $aSearchDirs);
 
 		require_once(APPROOT.'setup/setuputils.class.inc.php');

setup/applicationinstaller.class.inc.php

@@ -440,6 +440,12 @@ class ApplicationInstaller
 			// if the extensions dir exists, scan it for additional modules as well
 			$aDirsToScan[] = $sExtensionsPath;
 		}
+		$sExtraPath = APPROOT.'/data/'.$sEnvironment.'-modules/';
+		if (is_dir($sExtraPath))
+		{
+			// if the extra dir exists, scan it for additional modules as well
+			$aDirsToScan[] = $sExtraPath;
+		}
 		$sTargetPath = APPROOT.$sTargetDir;
 		if (!is_dir($sSourcePath))
 		{

setup/backup.class.inc.php

@@ -15,6 +15,69 @@
 //
 //   You should have received a copy of the GNU Affero General Public License
 //   along with iTop. If not, see <http://www.gnu.org/licenses/>
+/**
+ * Handles adding directories into a Zip archive
+ */
+class ZipArchiveEx extends ZipArchive
+{
+	public function addDir($sDir, $sZipDir = '')
+	{
+		if (is_dir($sDir))
+		{
+			if ($dh = opendir($sDir))
+			{
+				// Add the directory
+				if (!empty($sZipDir)) $this->addEmptyDir($sZipDir);
+					
+				// Loop through all the files
+				while (($sFile = readdir($dh)) !== false)
+				{
+					// If it's a folder, run the function again!
+					if (!is_file($sDir.$sFile))
+					{
+						// Skip parent and root directories
+						if (($sFile !== ".") && ($sFile !== ".."))
+						{
+							$this->addDir($sDir.$sFile."/", $sZipDir.$sFile."/");
+						}
+					}
+					else
+					{
+						// Add the files
+						$this->addFile($sDir.$sFile, $sZipDir.$sFile);
+					}
+				}
+			}
+		}
+	}
+	/**
+	 * Extract a whole directory from the archive.
+	 * Usage: $oZip->extractDirTo('/var/www/html/itop/data', '/production-modules/')
+	 * @param string $sDestinationDir
+	 * @param string $sZipDir Must start and end with a slash !!
+	 * @return boolean
+	 */
+	public function extractDirTo($sDestinationDir, $sZipDir)
+	{
+		$aFiles = array();
+		for($i = 0; $i < $this->numFiles; $i++)
+		{
+			$sEntry = $this->getNameIndex($i);
+			//Use strpos() to check if the entry name contains the directory we want to extract
+			if (strpos($sEntry, $sZipDir) === 0)
+			{
+				//Add the entry to our array if it in in our desired directory
+				$aFiles[] = $sEntry;
+			}
+		}
+		// Extract only the selcted files
+		if ((count($aFiles)  > 0) && ($this->extractTo($sDestinationDir, $aFiles) === true)) 
+		{
+			return true;
+		}
+		return false;
+	}
+} // class ZipArchiveEx
 
 
 class BackupException extends Exception
@@ -140,6 +203,15 @@ class DBBackup
 				'dest' => 'delta.xml',
 			);
 		}
+		$sExtraDir = APPROOT.'data/'.utils::GetCurrentEnvironment().'-modules/';
+		if (is_dir($sExtraDir))
+		{
+			$aContents[] = array(
+				'source' => $sExtraDir,
+				'dest' => utils::GetCurrentEnvironment().'-modules/',
+			);
+		}
+		
 		$this->DoZip($aContents, $sZipFile);
 		// Windows/IIS: the data file has been created by the spawned process...
 		//   trying to delete it will issue a warning, itself stopping the setup abruptely
@@ -249,7 +321,7 @@ class DBBackup
 		foreach ($aFiles as $aFile)
 		{
 			$sFile = $aFile['source'];
-			if (!is_file($sFile))
+			if (!is_file($sFile) && !is_dir($sFile))
 			{
 				throw new BackupException("File '$sFile' does not exist or could not be read");
 			}
@@ -258,13 +330,20 @@ class DBBackup
 		$sZipDir = dirname($sZipArchiveFile);
 		SetupUtils::builddir($sZipDir);
 	
-		$oZip = new ZipArchive();
+		$oZip = new ZipArchiveEx();
 		$res = $oZip->open($sZipArchiveFile, ZipArchive::CREATE | ZipArchive::OVERWRITE);
 		if ($res === TRUE)
 		{
 			foreach ($aFiles as $aFile)
 			{
-				$oZip->addFile($aFile['source'], $aFile['dest']);
+				if (is_dir($aFile['source']))
+				{
+					$oZip->addDir($aFile['source'], $aFile['dest']);
+				}
+				else
+				{
+					$oZip->addFile($aFile['source'], $aFile['dest']);
+				}
 			}
 			if ($oZip->close())
 			{

setup/runtimeenv.class.inc.php

@@ -323,6 +323,11 @@ class RunTimeEnvironment
 		{
 			$aDirsToCompile[] = APPROOT.'extensions';
 		}
+		$sExtraDir = APPROOT.'data/'.$this->sTargetEnv.'-modules/';
+		if (is_dir($sExtraDir))
+		{
+			$aDirsToCompile[] = $sExtraDir;
+		}
 		
 		$aRet = array();
 
@@ -352,9 +357,11 @@ class RunTimeEnvironment
 		foreach($aModules as $foo => $oModule)
 		{
 			$sModule = $oModule->GetName();
-			if (array_key_exists($sModule, $aAvailableModules))
+			$sModuleRootDir = $oModule->GetRootDir();
+			$bIsExtra = (strpos($sModuleRootDir, $sExtraDir) !== false);
+			if (array_key_exists($sModule, $aAvailableModules)) 
 			{
-				if ($aAvailableModules[$sModule]['version_db'] != '')
+				if (($aAvailableModules[$sModule]['version_db'] != '') ||  $bIsExtra) //Extra modules are always selected
 				{
 					$aRet[] = $oModule;
 				}
@@ -371,6 +378,14 @@ class RunTimeEnvironment
 		return $aRet;
 	}
 
+	/**
+	 * Compile the data model by imitating the given environment
+	 * The list of modules to be installed in the target environment is:
+	 *  - the list of modules present in the "source_dir" (defined by the source environment) which are marked as "installed" in the source environment's database
+	 *  - plus the list of modules present in the "extra" directory of the target environment: data/<target_environment>-modules/
+	 * @param string $sSourceEnv The name of the source environment to 'imitate'
+	 * @param bool $bUseSymLinks Whether to create symbolic links instead of copies
+	 */
 	public function CompileFrom($sSourceEnv, $bUseSymLinks = false)
 	{
 		$oSourceConfig = new Config(utils::GetConfigFilePath($sSourceEnv));
@@ -538,8 +553,14 @@ class RunTimeEnvironment
 		}
 	}
 	
-	public function RecordInstallation(Config $oConfig, $sDataModelVersion, $aSelectedModules, $sModulesRelativePath)
+	public function RecordInstallation(Config $oConfig, $sDataModelVersion, $aSelectedModules, $sModulesRelativePath, $sShortComment = null)
 	{
+		if ($sShortComment === null)
+		{
+			$sShortComment = 'Done by the setup program';
+		}
+		$sMainComment = $sShortComment."\nBuilt on ".ITOP_BUILD_DATE;
+		
 		// Record datamodel version
 		$aData = array(
 			'source_dir' => $oConfig->Get('source_dir'),
@@ -557,7 +578,7 @@ class RunTimeEnvironment
 		$oInstallRec = new ModuleInstallation();
 		$oInstallRec->Set('name', ITOP_APPLICATION);
 		$oInstallRec->Set('version', ITOP_VERSION.'.'.ITOP_REVISION);
-		$oInstallRec->Set('comment', "Done by the setup program\nBuilt on ".ITOP_BUILD_DATE);
+		$oInstallRec->Set('comment', $sMainComment);
 		$oInstallRec->Set('parent_id', 0); // root module
 		$oInstallRec->Set('installed', $iInstallationTime);
 		$iMainItopRecord = $oInstallRec->DBInsertNoReload();
@@ -572,7 +593,7 @@ class RunTimeEnvironment
 			$sName = $sModuleId;
 			$sVersion = $aModuleData['version_code'];
 			$aComments = array();
-			$aComments[] = 'Done by the setup program';
+			$aComments[] = $sShortComment;
 			if ($aModuleData['mandatory'])
 			{
 				$aComments[] = 'Mandatory';
@@ -697,4 +718,16 @@ class RunTimeEnvironment
 	{
 		SetupPage::log_ok($sText);
 	}
+	
+	public function GetCurrentDataModelVersion()
+	{
+		$oSearch = DBObjectSearch::FromOQL("SELECT ModuleInstallation WHERE name='".DATAMODEL_MODULE."'");
+		$oSet = new DBObjectSet($oSearch, array(), array('installed' => false));
+		$oLatestDM = $oSet->Fetch();
+		if ($oLatestDM == null)
+		{
+			return '0.0.0';
+		}
+		return $oLatestDM->Get('version');
+	}
 } // End of class

setup/setuputils.class.inc.php

@@ -1123,12 +1123,31 @@ EOF
 		{
 			$aDirsToScan[] = APPROOT.'extensions';
 		}
+		if (is_dir(APPROOT.'data'))
+		{
+			$aDirsToScan[] = APPROOT.'extensions';
+		}
 		if (is_dir($oWizard->GetParameter('copy_extensions_from')))
 		{
 			$aDirsToScan[] = $oWizard->GetParameter('copy_extensions_from');
 		}
+		$sExtraDir = APPROOT.'data/production-modules/';
+		if (is_dir($sExtraDir))
+		{
+			$aDirsToScan[] = $sExtraDir;
+		}
 		$oProductionEnv = new RunTimeEnvironment();
 		$aAvailableModules = $oProductionEnv->AnalyzeInstallation($oConfig, $aDirsToScan, $bAbortOnMissingDependency, $aModulesToLoad);
+		
+		foreach($aAvailableModules as $key => $aModule)
+		{
+			$bIsExtra = (array_key_exists('root_dir', $aModule) && (strpos($aModule['root_dir'], $sExtraDir) !== false)); // Some modules (root, datamodel) have no 'root_dir'
+			if ($bIsExtra)
+			{
+				// Modules in data/production-modules/ are considered as mandatory and always installed
+				$aAvailableModules[$key]['visible'] = false;
+			}
+		}
 
 		return $aAvailableModules;
 	}

synchro/synchrodatasource.class.inc.php

@@ -1824,6 +1824,13 @@ class SynchroReplica extends DBObject implements iDisplay
 	 */	
 	protected function UpdateObjectFromReplica($oDestObj, $aAttributes, $oChange, &$oStatLog, $sStatsCode, $sStatsCodeError)
 	{
+if (!is_object($oDestObj))
+{
+	IssueLog::Error('About to update a NON object in UpdateObjectFromReplica. Replica_id = '.$this->GetKey().' $oDestObj = '.var_export($oDestObj, true));
+	IssueLog::Error(MyHelpers::get_callstack_text());
+	return false;
+}
+
 		$aValueTrace = array();
 		$bModified = false;
 		try

webservices/itopsoap.examples.php

@@ -25,7 +25,7 @@
  */
 
 require_once('itopsoaptypes.class.inc.php');
-$sItopRoot = 'http'.(utils::IsConnectionSecure() ? 's' : '').'://'.$_SERVER['SERVER_NAME'].':'.$_SERVER['SERVER_PORT'].dirname($_SERVER['SCRIPT_NAME']).'/..';
+$sItopRoot = 'https://'.$_SERVER['SERVER_NAME'].':'.$_SERVER['SERVER_PORT'].dirname($_SERVER['SCRIPT_NAME']).'/..';
 $sWsdlUri = $sItopRoot.'/webservices/itop.wsdl.php';
 //$sWsdlUri .= '?service_category=';
 
@@ -58,19 +58,19 @@ try
 		'HW found shutdown', /* description */
 		null, /* caller */
 		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'Demo'))), /* customer */
-		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'NW Management'))), /* service */
+		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'Computers and peripherals'))), /* service */
 		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'Troubleshooting'))), /* service subcategory */
 		'', /* product */
-		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'NW support'))), /* workgroup */
+		new SOAPExternalKeySearch(array(new SOAPSearchCondition('name', 'Network support'))), /* workgroup */
 		array(
 			new SOAPLinkCreationSpec(
-				'Device',
-				array(new SOAPSearchCondition('name', 'switch01')),
+				'NetworkDevice',
+				array(new SOAPSearchCondition('name', 'Switch1')),
 				array()
 			),
 			new SOAPLinkCreationSpec(
 				'Server',
-				array(new SOAPSearchCondition('name', 'dbserver1.demo.com')),
+				array(new SOAPSearchCondition('name', 'Server1')),
 				array()
 			),
 		), /* impacted cis */