N°1806, N°1069: CSV and Excel export and import of documents (files) and images as URLs.

git-svn-id: http://svn.code.sf.net/p/itop/code/trunk@4952 a333f486-631f-4898-b8df-5754b55c2be0

application/utils.inc.php

@@ -1590,4 +1590,211 @@ class utils
 		}
 		return null;
 	}
+	
+	/**
+	 * Check if the given path/url is an http(s) URL
+	 * @param string $sPath
+	 * @return boolean
+	 */
+	public static function IsURL($sPath)
+	{
+		$bRet = false;
+		if ((substr($sPath, 0, 7) == 'http://') || (substr($sPath, 0, 8) == 'https://') || (substr($sPath, 0, 8) == 'ftp://'))
+		{
+			$bRet = true;
+		}
+		return $bRet;
+	}
+	
+	/**
+	 * Check if the given URL is a link to download a document/image on the CURRENT iTop
+	 * In such a case we can read the content of the file directly in the database (if the users rights allow) and return the ormDocument
+	 * @param string $sPath
+	 * @return false|ormDocument
+	 * @throws Exception
+	 */
+	public static function IsSelfURL($sPath)
+	{
+		$result = false;
+		$sPageUrl = utils::GetAbsoluteUrlAppRoot().'pages/ajax.document.php';
+		if (substr($sPath, 0, strlen($sPageUrl)) == $sPageUrl)
+		{
+			// If the URL is an URL pointing to this instance of iTop, then
+			// extract the "query" part of the URL and analyze it
+			$sQuery = parse_url($sPath, PHP_URL_QUERY);
+			if ($sQuery !== null)
+			{
+				$aParams = array();
+				foreach(explode('&', $sQuery) as $sChunk)
+				{
+					$aParts = explode('=', $sChunk);
+					if (count($aParts) != 2) continue;
+					$aParams[$aParts[0]] = urldecode($aParts[1]);
+				}
+				$result = array_key_exists('operation', $aParams) && array_key_exists('class', $aParams) && array_key_exists('id', $aParams) && array_key_exists('field', $aParams) && ($aParams['operation'] == 'download_document');
+				if ($result)
+				{
+					// This is a 'download_document' operation, let's retrieve the document directly from the database
+					$sClass = $aParams['class'];
+					$iKey = $aParams['id'];
+					$sAttCode = $aParams['field'];
+
+					$oObj = MetaModel::GetObject($sClass, $iKey, false /* must exist */); // Users rights apply here !!
+					if ($oObj)
+					{
+						/**
+						 * @var ormDocument $result
+						 */
+						$result = clone $oObj->Get($sAttCode);
+						return $result;
+					}
+				}
+			}
+			throw new Exception('Invalid URL. This iTop URL is not pointing to a valid Document/Image.');
+		}
+		return $result;
+	}
+	
+	/**
+	 * Read the content of a file (and retrieve its MIME type) from either:
+	 * - an URL pointing to a blob (image/document) on the current iTop server
+	 * - an http(s) URL
+	 * - the local file system (but only if you are an administrator)
+	 * @param string $sPath
+	 * @return string[]|NULL[]
+	 * @throws Exception
+	 */
+	public static function FileGetContentsAndMIMEType($sPath)
+	{
+		$oUploadedDoc = null;
+		$aKnownExtensions = array(
+				'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+				'xltx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.template',
+				'potx' => 'application/vnd.openxmlformats-officedocument.presentationml.template',
+				'ppsx' => 'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
+				'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+				'sldx' => 'application/vnd.openxmlformats-officedocument.presentationml.slide',
+				'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+				'dotx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
+				'xlam' => 'application/vnd.ms-excel.addin.macroEnabled.12',
+				'xlsb' => 'application/vnd.ms-excel.sheet.binary.macroEnabled.12.xlsx',
+				'xltx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.template',
+				'potx' => 'application/vnd.openxmlformats-officedocument.presentationml.template',
+				'ppsx' => 'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
+				'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+				'sldx' => 'application/vnd.openxmlformats-officedocument.presentationml.slide',
+				'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+				'dotx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
+				'xlam' => 'application/vnd.ms-excel.addin.macroEnabled.12',
+				'xlsb' => 'application/vnd.ms-excel.sheet.binary.macroEnabled.12',
+				'jpg' => 'image/jpeg',
+				'jpeg' => 'image/jpeg',
+				'gif' => 'image/gif',
+				'png' => 'image/png',
+				'pdf' => 'application/pdf',
+				'doc' => 'application/msword',
+				'dot' => 'application/msword',
+				'xls' => 'application/vnd.ms-excel',
+				'ppt' => 'application/vnd.ms-powerpoint',
+				'vsd' => 'application/x-visio',
+				'vdx' => 'application/visio.drawing',
+				'odt' => 'application/vnd.oasis.opendocument.text',
+				'ods' => 'application/vnd.oasis.opendocument.spreadsheet',
+				'odp' => 'application/vnd.oasis.opendocument.presentation',
+				'zip' => 'application/zip',
+				'txt' => 'text/plain',
+				'htm' => 'text/html',
+				'html' => 'text/html',
+				'exe' => 'application/octet-stream'
+		);
+	
+		$sData = null;
+		$sMimeType = 'text/plain'; // Default MIME Type: treat the file as a bunch a characters...
+		$sFileName = 'uploaded-file'; // Default name for downloaded-files
+		$sExtension = '.txt'; // Default file extension in case we don't know the MIME Type
+	
+		if (static::IsURL($sPath))
+		{
+			if ($oUploadedDoc = static::IsSelfURL($sPath))
+			{
+				// Nothing more to do, we've got it !!
+			}
+			else
+			{
+				// Remote file, let's use the HTTP headers to find the MIME Type
+				$sData = @file_get_contents($sPath);
+				if ($sData === false)
+				{
+					throw new Exception("Failed to load the file from the URL '$sPath'.");
+				}
+				else
+				{
+					if (isset($http_response_header))
+					{
+						$aHeaders = static::ParseHeaders($http_response_header);
+						$sMimeType = array_key_exists('Content-Type', $aHeaders) ? strtolower($aHeaders['Content-Type']) : 'application/x-octet-stream';
+						// Compute the file extension from the MIME Type
+						foreach($aKnownExtensions as $sExtValue => $sMime)
+						{
+							if ($sMime === $sMimeType)
+							{
+								$sExtension = '.'.$sExtValue;
+								break;
+							}
+						}
+					}
+					$sFileName .= $sExtension;
+				}
+				$oUploadedDoc = new ormDocument($sData, $sMimeType, $sFileName);
+			}
+		}
+		else if (UserRights::IsAdministrator())
+		{
+			// Only administrators are allowed to read local files
+			$sData = @file_get_contents($sPath);
+			if ($sData === false)
+			{
+				throw new Exception("Failed to load the file '$sPath'. The file does not exist or the current process is not allowed to access it.");
+			}
+			$sExtension = strtolower(pathinfo($sPath, PATHINFO_EXTENSION));
+			$sFileName = basename($sPath);
+				
+			if (array_key_exists($sExtension, $aKnownExtensions))
+			{
+				$sMimeType = $aKnownExtensions[$sExtension];
+			}
+			else if (extension_loaded('fileinfo'))
+			{
+				$finfo = new finfo(FILEINFO_MIME);
+				$sMimeType = $finfo->file($sPath);
+			}
+			$oUploadedDoc = new ormDocument($sData, $sMimeType, $sFileName);
+		}
+		return $oUploadedDoc;
+	}
+	
+	protected static function ParseHeaders($aHeaders)
+	{
+		$aCleanHeaders = array();
+		foreach( $aHeaders as $sKey => $sValue )
+		{
+			$aTokens = explode(':', $sValue, 2);
+			if(isset($aTokens[1]))
+			{
+				$aCleanHeaders[trim($aTokens[0])] = trim($aTokens[1]);
+			}
+			else
+			{
+				// The header is not in the form Header-Code: Value
+				$aCleanHeaders[] = $sValue; // Store the value as-is
+				$aMatches = array();
+				// Check if it's not the HTTP response code
+				if( preg_match("|HTTP/[0-9\.]+\s+([0-9]+)|", $sValue, $aMatches) )
+				{
+					$aCleanHeaders['reponse_code'] = intval($aMatches[1]);
+				}
+			}
+		}
+		return $aCleanHeaders;
+	}
 }

core/attributedef.class.inc.php

@@ -5243,75 +5243,34 @@ class AttributeBlob extends AttributeDefinition
 	{
 		return '';
 	}
-
-
-	// Facilitate things: allow administrators to upload a document
-	// from a CSV by specifying its path/URL
+	
+	/**
+	 * Users can provide the document from an URL (including an URL on iTop itself)
+	 * for CSV import. Administrators can even provide the path to a local file
+	 * {@inheritDoc}
+	 * @see AttributeDefinition::MakeRealValue()
+	 */
 	public function MakeRealValue($proposedValue, $oHostObj)
 	{
+		if ($proposedValue === null) return null;
+		
 		if (is_object($proposedValue))
 		{
 			$proposedValue = clone $proposedValue;
 		}
 		else
 		{
-			if (file_exists($proposedValue) && UserRights::IsAdministrator())
+			try
 			{
-				$sContent = file_get_contents($proposedValue);
-				$sExtension = strtolower(pathinfo($proposedValue, PATHINFO_EXTENSION));
-				$sMimeType = "application/x-octet-stream";
-				$aKnownExtensions = array(
-						'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-						'xltx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.template',
-						'potx' => 'application/vnd.openxmlformats-officedocument.presentationml.template',
-						'ppsx' => 'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
-						'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-						'sldx' => 'application/vnd.openxmlformats-officedocument.presentationml.slide',
-						'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-						'dotx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
-						'xlam' => 'application/vnd.ms-excel.addin.macroEnabled.12',
-						'xlsb' => 'application/vnd.ms-excel.sheet.binary.macroEnabled.12.xlsx',
-						'xltx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.template',
-						'potx' => 'application/vnd.openxmlformats-officedocument.presentationml.template',
-						'ppsx' => 'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
-						'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-						'sldx' => 'application/vnd.openxmlformats-officedocument.presentationml.slide',
-						'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-						'dotx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
-						'xlam' => 'application/vnd.ms-excel.addin.macroEnabled.12',
-						'xlsb' => 'application/vnd.ms-excel.sheet.binary.macroEnabled.12',
-						'jpg' => 'image/jpeg',
-						'jpeg' => 'image/jpeg',
-						'gif' => 'image/gif',
-						'png' => 'image/png',
-						'pdf' => 'application/pdf',
-						'doc' => 'application/msword',
-						'dot' => 'application/msword',
-						'xls' => 'application/vnd.ms-excel',
-						'ppt' => 'application/vnd.ms-powerpoint',
-						'vsd' => 'application/x-visio',
-						'vdx' => 'application/visio.drawing', 
-						'odt' => 'application/vnd.oasis.opendocument.text',
-						'ods' => 'application/vnd.oasis.opendocument.spreadsheet',
-						'odp' => 'application/vnd.oasis.opendocument.presentation',
-						'zip' => 'application/zip',
-						'txt' => 'text/plain',
-						'htm' => 'text/html',
-						'html' => 'text/html', 
-						'exe' => 'application/octet-stream'
-					);
-
-				if (!array_key_exists($sExtension, $aKnownExtensions) && extension_loaded('fileinfo'))
-				{
-					$finfo = new finfo(FILEINFO_MIME);
-					$sMimeType = $finfo->file($proposedValue);
-				}
-				return new ormDocument($sContent, $sMimeType);			
+				// Read the file from iTop, an URL (or the local file system - for admins only)
+				$proposedValue = Utils::FileGetContentsAndMIMEType($proposedValue);
 			}
-			else
+			catch(Exception $e)
 			{
-				return new ormDocument($proposedValue, 'text/plain');
-			}
+				IssueLog::Warning(get_class($this)."::MakeRealValue - ".$e->getMessage());
+				// Not a real document !! store is as text !!! (This was the default behavior before)
+				$proposedValue = new ormDocument($e->getMessage()." \n".$proposedValue, 'text/plain');
+			}	
 		}
 		return $proposedValue;
 	}
@@ -5426,6 +5385,11 @@ class AttributeBlob extends AttributeDefinition
 
 	public function GetAsCSV($sValue, $sSeparator = ',', $sTextQualifier = '"', $oHostObject = null, $bLocalize = true, $bConvertToPlainText = false)
 	{
+		$sAttCode = $this->GetCode();
+		if ($sValue instanceof ormDocument)
+		{
+			return $sValue->GetDownloadURL(get_class($oHostObject), $oHostObject->GetKey(), $sAttCode);
+		}
 		return ''; // Not exportable in CSV !
 	}
 	
@@ -5529,39 +5493,31 @@ class AttributeImage extends AttributeBlob
 {
 	public function GetEditClass() {return "Image";}
 
-	// Facilitate things: allow administrators to upload a document
-	// from a CSV by specifying its path/URL
+	/**
+	 * {@inheritDoc}
+	 * @see AttributeBlob::MakeRealValue()
+	 */
 	public function MakeRealValue($proposedValue, $oHostObj)
 	{
-		if (is_object($proposedValue))
-		{
-			$proposedValue = clone $proposedValue;
-		}
-		else
+		$oDoc = parent::MakeRealValue($proposedValue, $oHostObj);
+		// The validation of the MIME Type is done by CheckFormat below
+		return $oDoc;
+	}
+	
+	/**
+	 * Check that the supplied ormDocument actually contains an image
+	 * {@inheritDoc}
+	 * @see AttributeDefinition::CheckFormat()
+	 */
+	public function CheckFormat($value)
+	{
+		if ($value instanceof ormDocument)
 		{
-			if (file_exists($proposedValue) && UserRights::IsAdministrator())
-			{
-				$sContent = file_get_contents($proposedValue);
-				$sExtension = strtolower(pathinfo($proposedValue, PATHINFO_EXTENSION));
-				$sMimeType = "application/x-octet-stream";
-				$aKnownExtensions = array(
-					'jpg' => 'image/jpeg',
-					'jpeg' => 'image/jpeg',
-					'gif' => 'image/gif',
-					'png' => 'image/png'
-				);
-
-				if (!array_key_exists($sExtension, $aKnownExtensions) && extension_loaded('fileinfo'))
-				{
-					$finfo = new finfo(FILEINFO_MIME);
-					$sMimeType = $finfo->file($proposedValue);
-				}
-				return new ormDocument($sContent, $sMimeType);
-			}
+			return ($value->GetMainMimeType() == 'image');
 		}
-		return $proposedValue;
+		return true;
 	}
-
+	
 	public function GetAsHTML($value, $oHostObject = null, $bLocalize = true)
 	{
 		$iMaxWidthPx = $this->Get('display_max_width');
@@ -5570,7 +5526,16 @@ class AttributeImage extends AttributeBlob
 		$sRet = '<img src="'.$sUrl.'" style="max-width: '.$iMaxWidthPx.'px; max-height: '.$iMaxHeightPx.'px">';
 		if (is_object($value) && !$value->IsEmpty())
 		{
-			$sUrl = $value->GetDownloadURL(get_class($oHostObject), $oHostObject->GetKey(), $this->GetCode());
+			if ($oHostObject->IsNew() || ($oHostObject->IsModified() && (array_key_exists($this->GetCode(), $oHostObject->ListChanges()))))
+			{
+				// If the object is modified (or not yet stored in the database) we must serve the content of the image directly inline
+				// otherwise (if we just give an URL) the browser will be given the wrong content... and may cache it
+				$sUrl = 'data:'.$value->GetMimeType().';base64,'.base64_encode($value->GetData());
+			}
+			else
+			{
+				$sUrl = $value->GetDownloadURL(get_class($oHostObject), $oHostObject->GetKey(), $this->GetCode());
+			}
 			$sRet = '<img src="'.$sUrl.'" style="max-width: '.$iMaxWidthPx.'px; max-height: '.$iMaxHeightPx.'px">';
 		}
 		return '<div class="view-image" style="width: '.$iMaxWidthPx.'px; height: '.$iMaxHeightPx.'px;"><span class="helper-middle"></span>'.$sRet.'</div>';

core/excelbulkexport.class.inc.php

@@ -188,6 +188,11 @@ EOF
 				$oAttDef = MetaModel::GetAttributeDef(get_class($oObj), $sAttCode);
 				$sRet = $oAttDef->GetAsCSV($value, '', '', $oObj);
 			}
+			else if ($value instanceOf ormDocument)
+			{
+				$oAttDef = MetaModel::GetAttributeDef(get_class($oObj), $sAttCode);
+				$sRet = $oAttDef->GetAsCSV($value, '', '', $oObj);
+			}
 			else
 			{
 				$oAttDef = MetaModel::GetAttributeDef(get_class($oObj), $sAttCode);