<?php
/**
 * Import file model
 *
 * @package		CSVIVirtueMart
 * @author		Roland Dalmulder
 * @link		http://www.csvimproved.com
 * @copyright 	Copyright (C) 2006 - 2012 RolandD Cyber Produksi
 * @license 	GNU/GPLv3 http://www.gnu.org/licenses/gpl-3.0.html
 * @version		$Id: importfile.php 1892 2012-02-11 11:01:09Z RolandD $
 */

defined( '_JEXEC' ) or die( 'Direct Access to this location is not allowed.' );

jimport( 'joomla.application.component.model' );

/**
 * Import file Model
 *
 * @package CSVIVirtueMart
 */
class CsvivirtuemartModelImportfile extends JModel {

	// Private variables
	/** @var integer for keeping track when the script started */
	private $_starttime = 0;
	/** @var array - Indexed array of field names (mixed case) */
	private $_supported_fields = array();
	/** @var array - Associative array of field_name (lower case) => field default value */
	private $_field_defaults = array();
	/** @var array - Associative array of field_name (lower case) => field published? value */
	private $_field_published = array();
	/** @var integer the database ID for the vendor */
	private $_vendor_id = null;
	/** @var bool sets if the default value should be used or not */
	private $_skip_default_value = null;
	/** @var array contains a list of vendor currencies */
	private $_vendor_currencies = array();
	/** @var object contains the ICEcat helper */
	private $_icecat = null;

	// Protected variables
	/** @var object contains the data to import */
	protected $csvi_data = null;
	/** @var object contains the fields to import */
	protected $_csvifields = null;
	/** @var array contains the available fields */
	protected $_avfields = null;
	/** @var array contains the ICEcat data */
	protected $icecat_data = null;
	/** @var array contains the fields to combine */
	protected $combine_fields = array();
	/** @var array contains the combine settings */
	protected $combine_settings = array();
	/** @var string contains the name of the last import field */
	protected $_lastfield = null;

	/**
	 * Compile a list of helper files needed to include
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		public
	 * @param
	 * @return 		array	list of helper files to include
	 * @since 		3.0
	 */
	public function getHelperFiles() {
		$data = JRequest::getVar('jform', array(), 'post', 'array');
		$helpers = array();
		$upload_parts = array();

		// Get the file extension of the import file
		switch (strtolower($data['general']['loadfrom'])) {
			case 'fromupload':
				$upload = JRequest::getVar('import_file', '', 'files');
				if (isset($upload['name'])) $upload_parts = pathinfo($upload['name']);
				break;
			case 'fromserver':
				$upload_parts = pathinfo($data['general']['local_csv_file']);
				break;
			case 'fromurl':
				$upload_parts = pathinfo($data['general']['urlfile']);
				break;
			case 'fromftp':
				$upload_parts = pathinfo($data['general']['ftpfile']);
				break;
		}

		if (isset($upload_parts['basename']) && empty($upload_parts['basename'])) {
			JError::raiseNotice(0, JText::_('COM_CSVIVIRTUEMART_NO_FILE_UPLOADED'));
			return false;
		}

		// Set the file helper
		if (!array_key_exists('extension', $upload_parts)) {
			JError::raiseNotice(0, JText::_('COM_CSVIVIRTUEMART_NO_EXTENSION_FOUND'));
			return false;
		}
		else {
			switch ($upload_parts['extension']) {
				case 'xml':
					$helpers[] = $upload_parts['extension'];
					$fileclass = 'Xml';
					break;
				case 'xls':
					$helpers[] = $upload_parts['extension'];
					$helpers[] = 'excel_reader2';
					$fileclass = 'Xls';
					break;
				case 'ods':
					$helpers[] = $upload_parts['extension'];
					$helpers[] = 'ods_reader';
					$fileclass = 'Ods';
					break;
				default:
					// Treat any unknown type as CSV
					$helpers[] = 'csv';
					$fileclass = 'Csv';
					break;
			}
			// Set the file class name
			JRequest::setVar('fileclass', $fileclass.'File');

			// Do we need to load the image helper
			switch ($data['import_type']) {
				case 'productimport':
				case 'categorydetailsimport':
				case 'productfilesimport':
					$helpers[] = 'images';
					$helpers[] = 'vm_config';
					break;
			}

			// Add the helpers to the session
			$session = JFactory::getSession();
			$option = JRequest::getVar('option');
			$session->set($option.'.helper_files', serialize($helpers));

			return $helpers;
		}
	}

	/**
	 * Make preparations for the import
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo 		Fix storing of log when file cannot be retrieved from FTP
	 * @see
	 * @access 		public
	 * @param
	 * @return 		bool	true on file OK | false on file NOK
	 * @since 		3.0
	 */
	public function getPrepareImport() {
		// Get the form data
		$session = JFactory::getSession();
		$option = JRequest::getVar('option');
		$data 	= JRequest::getVar('jform', array(), 'post', 'array');

		// Re-order the replacement fields
		if (array_key_exists('replacement_import_fields', $data) && array_key_exists('_selected_name', $data['replacement_import_fields'])) {
			$fields = array();
			$replacetext = 0;
			$replaceregex = 0;
			foreach ($data['replacement_import_fields']['_selected_name'] as $rkey => $name) {
				if ($data['replacement_import_fields']['_replace_type'][$rkey] == 1) {
					$data['replacement_fields'][$name]['findregex'][$replaceregex] = $data['replacement_import_fields']['_old_value'][$rkey];
					$data['replacement_fields'][$name]['replaceregex'][$replaceregex] = $data['replacement_import_fields']['_new_value'][$rkey];
					$replaceregex++;
				}
				else {
					$data['replacement_fields'][$name]['findtext'][$replacetext] = $data['replacement_import_fields']['_old_value'][$rkey];
					$data['replacement_fields'][$name]['replacetext'][$replacetext] = $data['replacement_import_fields']['_new_value'][$rkey];
					$replacetext++;
				}
			}
		}

		// Set the template name
		$data['template_name'] = JRequest::getVar('template_name');

		$template = new CsviTemplate($data);
		JRequest::setVar('template', $template);
		$session->set($option.'.global.template', serialize($template));

		// Initiate the log
		$csvilog = new CsviLog();
		JRequest::setVar('csvilog', $csvilog);

		// Create a new Import ID in the logger
		$csvilog->setId();

		// Set to collect debug info
		$csvilog->setDebug($template->getValue('collect_debug_info', 'general'));

		// Retrieve the available fields
		$availablefields = $this->getModel('Availablefields');
		$this->_avfields = $availablefields->getAvailableFields($template->getValue('import_type'));
		$session->set($option.'.avfields', serialize($this->_avfields));
		// Needed for file class
		JRequest::setVar('avfields', $this->_avfields);

		// Process the file to import
		if ($template->getValue('im_mac', 'general', false)) {
			// Auto detect line-endings to also support Mac line-endings
			ini_set('auto_detect_line_endings', true);
		}
		$fileclass = JRequest::getVar('fileclass');
		$csvifile = new $fileclass;
		$csvifile->validateFile();
		$csvifile->processFile();
		if (!$csvifile->fp) {
			$this->getCleanSession();
			JError::raiseNotice(0, JText::_('COM_CSVIVIRTUEMART_CANNOT_READ_FILE'));
			return false;
		}
		else {
			// Load column headers
			if ($template->getValue('use_column_headers', 'general')) {
				$csvifile->loadColumnHeaders();
				$session->set($option.'.csvicolumnheaders', serialize(JRequest::getVar('columnheaders')));
			}
			else if ($template->getValue('skip_first_line', 'general')) {
				// Move 1 row forward as we are skipping the first line
				$csvifile->next();
			}

			// Set some log info
			$csvilog->SetAction('import');
			$csvilog->SetActionType($template->getValue('import_type'), $template->getValue('template_name'));

			// Load the fields
			if ($this->_retrieveConfigFields($csvifile)) $session->set($option.'.csvifields', serialize(JRequest::getVar('csvifields')));
			else {
				JError::raiseNotice(0, JText::_('COM_CSVIVIRTUEMART_CANNOT_LOAD_FIELDS'));
				return false;
			}

			// Write out some import settings
			$this->_importDetails();

			// Store the file position
			$session->set($option.'.filepos', serialize($csvifile->getFilePos()));

			// Empty the data first so we don't break the session
			$csvifile->clearData();

			// Store the CSVI file handler
			$session->set($option.'.csvifile', serialize($csvifile));

			// Store the CSVI log handler
			$session->set($option.'.csvilog', serialize($csvilog));

			// Store the preview handler
			$session->set($option.'.csvipreview', serialize($template->getValue('show_preview', 'general')));

			// Store the settings
			$settings = new CsvivirtuemartModelSettings();
			$params = $settings->getSettings();
			$session->set($option.'.csvisettings', serialize($params));

			// Set the combine separator
			$this->combine_settings['separator'] = ' ';
			$this->combine_settings['fieldname'] = null;

			// Unpublish any products if needed
			if ($template->getValue('unpublish_before_import', 'product', 0)) $this->_unpublishProducts();
			return true;
		}
	}

	/**
	 * Make preparations to do an import
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo 		Fix the setting of the file position on subsequent imports
	 * @see 		finishProcess
	 * @access 		public
	 * @param
	 * @return
	 * @since 		3.0
	 */
	public function getDoImport() {
		// Set the system limits
		$this->_systemLimits();

		// Set Mac line-ending support if needed
		$template = JRequest::getVar('template');
		if ($template->getValue('im_mac', 'general', false)) {
			// Auto detect line-endings to also support Mac line-endings
			ini_set('auto_detect_line_endings', true);
		}

		// Open the file
		$csvifile = JRequest::getVar('csvifile');
		if ($csvifile->processFile()) {
			// Set the file pointer
			$session = JFactory::getSession();
			$option = JRequest::getVar('option');
			$csvifile->setFilePos(unserialize($session->get($option.'.filepos', 0)));

			// Set the line counter
			JRequest::setVar('currentline', 1);

			// Set the fields found in the file
			$this->_csvifields = JRequest::getVar('csvifields');
			$this->_avfields = JRequest::getVar('avfields');

			return true;
		}
		else return false;
	}

	/**
	 * Start the import
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo		Separate view for preview
	 * @todo		Rewrite memory usage for debug
	 * @see
	 * @access 		public
	 * @param
	 * @return
	 * @since 		3.0
	 */
	public function getProcessData() {
		// Set some variables
		$data_preview = array();
		$processdata = true;
		$redirect = false;

		// Load the log
		$csvilog = JRequest::getVar('csvilog');

		// Load the settings
		$settings_model = new CsvivirtuemartModelSettings();

		// Load the template
		$template = JRequest::getVar('template');

		// Load the file
		$csvifile = JRequest::getVar('csvifile');

		// Load the import routine
		$classname = $this->getModel($template->getValue('import_type'));
		$routine = new $classname;

		// Start processing data
		while ($processdata) {
			// Pass the total log line to the logger
			$csvilog->setLinenumber((JRequest::getInt('currentline')+JRequest::getInt('totalline')));

			// If the number of lines is set to 0, do unlimited import
			if (($settings_model->getSetting('import_nolines', 0) == 0) || JRequest::getBool('cron', false)) {
				$nolines = JRequest::getInt('currentline')+1;
			}
			else $nolines = $settings_model->getSetting('import_nolines');

			if (JRequest::getInt('currentline') <= $nolines) {
				// For XML files, it may be necessary to refresh the headers before reading the next record
				if ($csvifile->extension == 'xml' && $template->getValue('refresh_xml_headers', 'general')) {
					$csvifile->loadColumnHeaders();
					if ($this->_retrieveConfigFields() == false) {
						// Error found - Finish processing
						$redirect = $this->finishProcess(false);
						$processdata = false;
						continue;
					}
					else $routine->refresh();
				}
				// Load the data
				$csvi_data = $csvifile->ReadNextLine();
				if ($csvi_data == false) {
					if (JRequest::getBool('csvipreview')) {
						// Set the headers
						$headers = array();
						foreach ($this->_csvifields as $fieldname => $value) {
							if ($value['published']) {
								if (isset($routine->$fieldname) || empty($routine->$fieldname)) $headers[] = $fieldname;
							}
						}
						JRequest::setVar('headers_preview', $headers);

						// Set the data
						JRequest::setVar('data_preview', $data_preview);

						// Clean the session
						$this->getCleanPreview();

						$processdata = false;
						continue;
					}
					else {
						// Finish processing
						$this->finishProcess(true);
						$processdata = false;
					}
				}
				else {
					// Check if we need to add any extra fields
					if (count($this->_csvifields) > count($this->csvi_data)) {
						foreach ($this->_csvifields as $fieldname => $details) {
							if (!array_key_exists($details['order'], $csvi_data)) {
								if (!empty($details['default_value'])) {
									$csvi_data[$details['order']] = $details['default_value'];
								}
							}
						}
					}
					JRequest::setVar('csvi_data', $csvi_data);

					if ($this->_checkLimits()) {
						// Notify the debug log what line we are one
						$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DEBUG_PROCESS_LINE', (JRequest::getInt('currentline')+JRequest::getInt('totalline'))));

						// Start processing record
						if ($routine->getStart()) {
							if (JRequest::getBool('csvipreview')) {
								$this->loadSettings();
								// Update preview data
								foreach ($this->_csvifields as $fieldname => $value) {
									if ($value['published']) {
										if (isset($routine->$fieldname)) $preview_data[$value['order']] = $routine->$fieldname;
										else if (empty($routine->$fieldname)) $preview_data[$value['order']] = '';
									}
								}
								$data_preview[JRequest::getVar('currentline')] = $preview_data;

								if (JRequest::getInt('currentline') == $settings_model->getSetting('import_preview', 5)) {
									// Set the headers
									$headers = array();
									foreach ($this->_csvifields as $fieldname => $value) {
										if ($value['published']) {
											if (isset($routine->$fieldname) || empty($routine->$fieldname)) $headers[] = $fieldname;
										}
									}
									JRequest::setVar('headers_preview', $headers);

									// Set the data
									JRequest::setVar('data_preview', $data_preview);

									// Clean the session
									$this->getCleanPreview();

									$processdata = false;
									continue;
								}
							}
							else {
								// Now we import the rest of the records
								$routine->getProcessRecord();
							}

							// Increase the number of records processed
							JRequest::setVar('recordsprocessed', JRequest::getInt('recordsprocessed')+1);
						}
						// Increase linenumber
						JRequest::setVar('currentline', JRequest::getInt('currentline')+1);
					}
					else {
						// Finish processing
						$this->finishProcess(false);

						// Stop from processing any further, no time left
						$processdata = false;
					}
				}
			}
			// Prepare for page reload
			else {
				// Finish processing
				$this->finishProcess(false);

				// Stop from processing any further, no time left
				$processdata = false;
			}
		}

		// Post Processing
		if ($template->getValue('import_type') == 'productimport') {
			$routine->getPostProcessing(array_keys($this->_csvifields));
		}
	}

	/**
	 * Clean the session
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		public
	 * @param
	 * @return 		void
	 * @since 		3.0
	 */
	public function getCleanSession() {
		// Store the log results first
		$log = $this->getModel('log');
		$log->getStoreLogResults();

		// Get session handler
		$session = JFactory::getSession();
		$option = JRequest::getVar('option');

		// Unset all session values
		$session->clear($option.'.csvicolumnheaders');
		$session->clear($option.'.csvifields');
		$session->clear($option.'.avfields');
		$session->clear($option.'.csvifile');
		$session->clear($option.'.filepos');
		$session->clear($option.'.recordsprocessed');
		$session->clear($option.'.template_id');
		$session->clear($option.'.totalline');
		$session->clear($option.'.csvilog');
		$session->clear($option.'.global.template');
		$session->clear($option.'.csvisettings');
		$session->clear($option.'.helper_files');
	}

	/**
	 * Clean the session after preview
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		public
	 * @param
	 * @return 		void
	 * @since 		3.0
	 */
	private function getCleanPreview() {
		// Get session handler
		$session = JFactory::getSession();
		$option = JRequest::getVar('option');

		// Load the correct position
		if (JRequest::getBool('csvipreview')) {
			$template = JRequest::getVar('template');
			$csvifile = JRequest::getVar('csvifile');

			// Move back to the beginning
			$csvifile->rewind();

			// Move 1 row forward as this is the column header
			if ($template->getValue('use_column_headers', 'general') || $template->getValue('skip_first_line', 'general')) {
				$csvifile->next(true);
			}

			// Get the current position
			$filepos = $csvifile->getFilePos();
		}
		else $filepos = 0;

		// Unset all session values
		$session->set($option.'.filepos', serialize($filepos));
		$session->set($option.'.recordsprocessed', serialize(0));
		$session->set($option.'.csvipreview', serialize(false));
	}

	/**
	 * Sets the system limits to user defined values
	 *
	 * Sets the system limits to user defined values to allow for longer and
	 * bigger uploaded files
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo 		Allow 0 or -1 value
	 * @see
	 * @access 		private
	 * @param
	 * @return
	 * @since 		3.0
	 */
	private function _systemLimits() {
		// Set the start time of the script
		$this->_starttime = time();

		// Get the logger
		$csvilog = JRequest::getVar('csvilog');
		$template = JRequest::getVar('template');

		// See if we need to use the new limits
		if ($template->getValue('use_system_limits', 'limit')) {
			$csvilog->addDebug('Setting system limits:');
			// Apply the new memory limits
			$csvilog->addDebug('Setting max_execution_time to '.$template->getValue('max_execution_time').' seconds');
			@ini_set('max_execution_time', $template->getValue('max_execution_time'));
			if ($template->getValue('memory_limit', 'limit') == '-1') {
				$csvilog->addDebug('Setting memory_limit to '.$template->getValue('memory_limit', 'limit'));
				@ini_set('memory_limit', $template->getValue('memory_limit', 'limit'));
			}
			else {
				$csvilog->addDebug('Setting memory_limit to '.$template->getValue('memory_limit', 'limit').'M');
				@ini_set('memory_limit', $template->getValue('memory_limit', 'limit').'M');
			}
			$csvilog->addDebug('Setting post_max_size to '.$template->getValue('post_max_size', 'limit').'M');
			@ini_set('post_max_size', $template->getValue('post_max_size', 'limit').'M');
			$csvilog->addDebug('Setting upload_max_filesize to '.$template->getValue('upload_max_filesize', 'limit').'M');
			@ini_set('upload_max_filesize', $template->getValue('upload_max_filesize', 'limit').'M');
		}
	}

	/**
	 * Function to check if execution time is going to be passed.
	 *
	 * Memory can only be checked if the function memory_get_usage is available.
	 * If the function is not available always return true. This could lead to
	 * out of memory failure.
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see 		http://www.php.net/memory_get_usage
	 * @access 		private
	 * @param
	 * @return 		bool true when limits are not reached|false when limit is reached
	 * @since 		3.0
	 */
	private function _checkLimits() {
		$csvilog = JRequest::getVar('csvilog');

		// Check for maximum execution time
		$timepassed = time() - $this->_starttime;
		if (($timepassed + 5) > ini_get('max_execution_time') && ini_get('max_execution_time') > 0) {
			$csvilog->AddStats('information', JText::sprintf('COM_CSVIVIRTUEMART_MAXIMUM_EXECUTION_LIMIT_EXCEEDED', $timepassed));
			return false;
		}

		// Check for maximum memory usage
		if (!function_exists('memory_get_usage')) return true;
		else {
			// Get the size of the statistics
			$statslength = 0;
			if (isset($csvilog->stats)) {
				foreach ($csvilog->stats as $type => $value) {
					if (isset($value['message'])) $statslength += strlen($value['message']);
				}
			}
			$statslength = round($statslength/(1024*1024));

			// Get the size of the debug message
			$debuglength = round(strlen($csvilog->debug_message)/(1024*1024));

			// Get the size of the active memory in use
			$activelength = round(memory_get_usage()/(1024*1024));

			// Combine memory limits
			$totalmem = $activelength + $statslength + $debuglength;

			// Set the memory limit
			JRequest::setVar('maxmem', $totalmem);

			// Check if we are passing the memory limit
			if (($totalmem * 1.5) > (int)ini_get('memory_limit')) {
				$csvilog->AddStats('information', JText::_('COM_CSVIVIRTUEMART_MAXIMUM_MEMORY_LIMIT_EXCEEDED', $totalmem));
				return false;
			}

			// All is good return true
			return true;
		}
	}

	/**
	 * Print out import details
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		private
	 * @param
	 * @return
	 * @since 		3.0
	 */
	private function _importDetails() {
		// Get the logger
		$csvilog = JRequest::getVar('csvilog');
		// Get the template settings to see if we need a preview
		$template = JRequest::getVar('template');

		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_CSVI_VERSION_TEXT').JText::_('COM_CSVIVIRTUEMART_CSVI_VERSION'));
		if (function_exists('phpversion')) $csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_PHP_VERSION', phpversion()));

		// General settings
		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_GENERAL_SETTINGS'));
		// Show which template is used
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_TEMPLATE_NAME', $template->getValue('template_name')));
		// Auto detect delimiters
		$auto_detect = ($template->getValue('auto_detect_delimiters', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_AUTO_DETECT_DELIMITERS', $auto_detect));
		if ($auto_detect == JText::_('COM_CSVIVIRTUEMART_NO')) {
			// Check delimiter char
			$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_USING_DELIMITER', $template->getValue('field_delimiter', 'general')));
			// Check enclosure char
			$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_USING_ENCLOSURE', $template->getValue('text_enclosure', 'general')));
		}

		// Show import settings
		// Show template type
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_CHOSEN_IMPORT_TYPE', JText::_($template->getValue('import_type'))));

		// Use column headers as configuration
		$use_header = ($template->getValue('use_column_headers', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_USE_HEADER', $use_header));

		// Refresh xml headers for every record
		$refresh_xml_headers = ($template->getValue('refresh_xml_headers', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_REFRESH_XML_HEADER', $use_header));

		// Skip first line
		$skip_first = ($template->getValue('skip_first_line', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_SKIP_FIRST_LINE', $skip_first));

		// Ignore non-existing products
		$ignore_non_exist = ($template->getValue('ignore_non_exist', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_IGNORE_NON_EXIST', $ignore_non_exist));

		// Overwrite existing data
		$overwrite = ($template->getValue('overwrite_existing_data', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_OVERWRITE_EXISTING_DATA', $overwrite));

		// Skip default value
		$skip_default = ($template->getValue('skip_default_value', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_SKIP_DEFAULT_VALUE', $skip_default));

		// Show preview
		$use_preview = ($template->getValue('show_preview', 'general')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_USING_PREVIEW', $use_preview));

		// Products
		// Unpublish products before import
		$unpublish = ($template->getValue('unpublish_before_import', 'product')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_UNPUBLISH_BEFORE_IMPORT', $unpublish));


		// Categories
		// Append categories
		$append_cats = ($template->getValue('append_categories', 'category')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_APPEND_CATEGORIES', $append_cats));

		// Images
		// General options
		// Create image name
		$create_name = ($template->getValue('auto_generate_image_name', 'image')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_CREATE_IMAGE_NAME', $create_name));

		// Generate image name
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_IMAGE_BASED_ON', $template->getValue('type_generate_image_name', 'image')));

		// Image name format
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_IMAGE_NAME_FORMAT', $template->getValue('auto_generate_image_name_ext', 'image')));

		// Full image
		// Convert image
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_CONVERT_IMAGE', $template->getValue('convert_type', 'image')));

		// Save images on server
		$on_server = ($template->getValue('save_images_on_server', 'image')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_SAVE_IMAGES_ON_SERVER', $on_server));

		// Thumbnail image
		// Automatic thumbnail creation
		$auto_thumb = ($template->getValue('thumb_create', 'image')) ? JText::_('COM_CSVIVIRTUEMART_YES') : JText::_('COM_CSVIVIRTUEMART_NO');
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_AUTOMATIC_THUMBS', $auto_thumb));

		if ($auto_thumb == JText::_('COM_CSVIVIRTUEMART_YES')) {
			// Thumbnail format
			$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_FORMAT_THUMBS', $template->getValue('thumb_extension', 'image')));

			// Thumbnail width x height
			$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DIMENSION_THUMBS', $template->getValue('thumb_width', 'image'), $template->getValue('thumb_height', 'image')));
		}

		// Show the file paths
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DEBUG_FILE_PATH_PRODUCT_IMAGES', $template->getValue('file_location_product_images', 'path')));
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DEBUG_FILE_PATH_CATEGORY_IMAGES', $template->getValue('file_location_category_images', 'path')));
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DEBUG_FILE_PATH_MEDIA', $template->getValue('file_location_media', 'path')));

		// Show the max execution time
		$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_DEBUG_MAX_EXECUTION_TIME', ini_get('max_execution_time')));
	}

	/**
	 * Unpublish products before import
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see 		prepareImport()
	 * @access 		private
	 * @param
	 * @return
	 * @since 		3.0
	 */
	private function _unpublishProducts() {
		$db = JFactory::getDBO();
		$csvilog = JRequest::getVar('csvilog');

		$q = "UPDATE #__vm_product SET product_publish = 'N'";
		$db->setQuery($q);
		if ($db->query()) $csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_PRODUCT_UNPUBLISH_BEFORE_IMPORT'));
		else $csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_COULD_NOT_UNPUBLISH_BEFORE_IMPORT'), true);
	}

	/**
	* Builds arrays of field names and default values to be used during the creation of the headers list
	* The creation of the headers from the data file may need to be carried out for every row when processing
	* XML files and so efficiency is important for performance.
	*
	* Note: The array supported_fields should not be used as the basis for these arrays because it is a list of
	*   all available fields and some of these fields may not be mapped in the template.
	*/
	public function _fieldArrays() {
		$template = JRequest::getVar('template');
		$db = JFactory::getDBO();
		$this->_supportedfields = array();
		$this->_field_defaults = array();
		$this->_field_published = array();

		if ($template->getValue('use_column_headers', 'general')) {
			$supportedfields = array_flip($this->_avfields);
			foreach ($supportedfields as $name => $value) {
				$this->_supported_fields[] = $name;
				$this->_field_defaults[strtolower($name)] = null;
				$this->_field_published[strtolower($name)] = 1;
			}
		}
		// Use the template fields assigned to the template
		else {
			$import_fields = $template->getValue('import_fields');
			if (isset($import_fields['_selected_name'])) {
				$count = count($import_fields['_selected_name']);
				for ($rows = 0; $rows < $count; $rows++) {
					$this->_supported_fields[] = $import_fields['_selected_name'][$rows];
					$this->_field_defaults[strtolower($import_fields['_selected_name'][$rows])] = $import_fields['_default_value'][$rows];
					$this->_field_published[strtolower($import_fields['_selected_name'][$rows])] = $import_fields['_process_field'][$rows];
				}
			}
		}
		// Create the inverted array used to lookup the field name using lowercase
		$this->_supportedfields = array();
		foreach( $this->_supported_fields as $key => $value ) {
			$this->_supportedfields[strtolower($value)] = $key;
		}
	}

	/**
	 * Get the configuration fields the user wants to use
	 *
	 * The configuration fields can be taken from the uploaded file or from
	 * the database. Depending on the template settings.
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo 		Expand the no support fields message on line 398
	 * @see 		Templates()
	 * @access 		private
	 * @param
	 * @return 		bool true|false true when there are config fields|false when there are no or unsupported fields
	 * @since 		3.0
	 */
	private function _retrieveConfigFields($csvifile=false) {
		$db = JFactory::getDBO();
		$template = JRequest::getVar('template');
		$csvilog = JRequest::getVar('csvilog');
		if (!$csvifile) $csvifile = JRequest::getVar('csvifile');
		if (empty($this->_supportedfields)) $this->_fieldArrays();
		$columnheaders = JRequest::getVar('columnheaders');
		$csvifields = array();
		$nosupport = array();
		$csvilog->addDebug(JTEXT::_('COM_CSVIVIRTUEMART_DEBUG_LOAD_CONFIG_FIELDS'));

		if ($template->getValue('use_column_headers', 'general')) {
			// The user has column headers in the file
			JRequest::setVar('error_found', false);
			if ($columnheaders) {
				foreach ($columnheaders as $order => $name) {
					// Trim the name in case the name contains any preceding or trailing spaces
					$name = strtolower(trim($name));

					// Check if the fieldname is supported
					// No special field checking for Product Type Names upload
					if (array_key_exists($name, $this->_supportedfields)) {
						$csvilog->addDebug('Field: '.$name);
						$mixed_name = $this->_supported_fields[$this->_supportedfields[$name]];
						$csvifields[$mixed_name]['name'] = $mixed_name;
						$csvifields[$mixed_name]['order'] = $order;
						$csvifields[$mixed_name]['default_value'] = (array_key_exists($name, $this->_field_defaults)) ? $this->_field_defaults[$name] : null;
						$csvifields[$mixed_name]['published'] = (array_key_exists($name, $this->_field_published)) ? $this->_field_published[$name] : 'Y';
						$csvifields[$mixed_name]['combine'] = false;
					}
					else {
						// Check if the user has any field that is not supported
						if (strlen($name) == 0) $name = JText::_('COM_CSVIVIRTUEMART_FIELD_EMPTY');

						// Field is not supported, let's skip it
						$csvifields[$name]['name'] = $name;
						$csvifields[$name]['order'] = $order;
						$csvifields[$name]['default_value'] = null;
						$csvifields[$name]['published'] = 'N';
						$csvifields[$name]['combine'] = false;

						// Unset the column header so we can check if any fields are left over
						unset($columnheaders[$order]);

						// Collect the fieldnames to report them
						$nosupport[] = $name;
					}
				}
				if (empty($columnheaders)) {
					$csvilog->AddStats('incorrect', JText::_('COM_CSVIVIRTUEMART_NO_COLUMN_HEADERS_FOUND'));
					JRequest::setVar('error_found', true);
					return false;
				}

				if (!empty($nosupport)) {
					// Ensure the error message matches the file type
					switch($csvifile->extension) {
						case 'xml':
						case 'sql':
							$csvilog->AddStats('nosupport', implode(',', $nosupport).JText::_('COM_CSVIVIRTUEMART_FIELD_NOT_INCLUDED'));
							break;
						default:
							$csvilog->AddStats('nosupport', JText::sprintf('COM_CSVIVIRTUEMART_NO_SUPPORT', '<ul><li>'.implode('</li><li>', $nosupport).'</li></ul>'));
							break;
					}
					$csvilog->AddStats('information', JText::_('COM_CSVIVIRTUEMART_UNSUPPORTED_FIELDS'));
				}

				$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_USING_FILE_FOR_CONFIGURATION'));
			}
			else {
				$csvilog->AddStats('incorrect', JText::_('COM_CSVIVIRTUEMART_NO_COLUMN_HEADERS_FOUND'));
				JRequest::setVar('error_found', true);
				return false;
			}
		}
		// Use the fields assigned to the template
		else {
			$fields = $template->getValue('import_fields');

			if (!empty($fields)) {
				foreach ($fields['_selected_name'] as $fid => $name) {
					// Check if we are handling a combine field
					if ($name == 'combine') $name .= $fid;

					// Collect the data
					$csvifields[$name]['name'] = $name;
					$csvifields[$name]['order'] = $fid;
					$csvifields[$name]['default_value'] = $fields['_default_value'][$fid];
					$csvifields[$name]['published'] = $fields['_process_field'][$fid];
					$csvifields[$name]['combine'] = $fields['_combine_field'][$fid];

					if (!$csvifields[$name]['published']) $name .= ' ('.JText::_('COM_CSVIVIRTUEMART_FIELD_SKIPPED').')';
					$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_IMPORT_FIELD', $name));
				}
				$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_USE_DATABASE_FOR_CONFIGURATION'));
			}
			else {
				$csvilog->AddStats('incorrect', JText::_('NO_COLUMN_HEADERS_FOUND'));
				return false;
			}
		}

		// Make the fields to process global
		JRequest::setVar('csvifields', $csvifields);
		return true;
	}

	/**
	 * Handle the end of the import
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		private
	 * @param
	 * @return
	 * @since 		3.0
	 */
	public function finishProcess($finished=false) {
		$template = JRequest::getVar('template');
		$csvifile = JRequest::getVar('csvifile');
		$csvilog = JRequest::getVar('csvilog');

		// Adjust the current line, since it is not processing
		JRequest::setVar('currentline', JRequest::getInt('currentline')-1);

		// Session init
		$session = JFactory::getSession();
		$option = JRequest::getVar('option');
		if ($finished) {
			// Close the file
			if (is_object($csvifile)) $csvifile->closeFile(true);

			// Clean the session
			$this->getCleanSession();

			// Add the ID to the session as we need it for the redirect to the result page
			$session->set($option.'.run_id', $csvilog->getId());
		}
		else {
			// Flush the log details
			// Store the log results first
			$log = $this->getModel('log');
			$log->getStoreLogResults();

			// Create session variables
			$session->set($option.'.global.template', serialize($template));
			$session->set($option.'.csvicolumnheaders', serialize(JRequest::getVar('columnheaders')));
			$session->set($option.'.csvifields', serialize($this->_csvifields));
			$session->set($option.'.csvifile', serialize($csvifile));
			$session->set($option.'.csvilog', serialize($csvilog));
			$session->set($option.'.filepos', serialize($csvifile->getFilePos()));
			$session->set($option.'.recordsprocessed', serialize(JRequest::getInt('recordsprocessed')));
			$session->set($option.'.totalline', serialize(JRequest::getInt('currentline') + JRequest::getInt('totalline')));

			// Close the file
			$csvifile->closeFile(false);
		}
	}

	/**
	 * Create a proxy for including other models
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param
	 * @return
	 * @since 		3.0
	 */
	protected function getModel($model) {
		return $this->getInstance($model, 'CsvivirtuemartModel');
	}

	/**
	 * Load some settings we need for the functions
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		private
	 * @param
	 * @return
	 * @since 		3.0
	 */
	protected function loadSettings() {
		// Load the settings
		$template = JRequest::getVar('template');
		$this->_csvifields = JRequest::getVar('csvifields');
		$this->_avfields = JRequest::getVar('avfields');
		$this->_skip_default_value = $template->getValue('skip_default_value', 'general');

		// Set the last field, needed for the combine function
		$this->_lastfield = end($this->_csvifields);
	}

	/**
	 * Load the data to import
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param
	 * @return
	 * @since 		3.4
	 */
	protected function loadData() {
		 $this->csvi_data = JRequest::getVar('csvi_data', '', 'default', 'none', 2);
	}

	/**
	 * Get the product id, this is necessary for updating existing products
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo 		Reduce number of calls to this function
	 * @see
	 * @access 		protected
	 * @param
	 * @return 		integer	product_id is returned
	 * @since 		3.0
	 */
	protected function getProductId() {
		$db = JFactory::getDBO();
		$csvilog = JRequest::getVar('csvilog');
		$template = JRequest::getVar('template');
		$update_based_on = $template->getValue('update_based_on', 'product', 'product_sku');
		switch ($update_based_on) {
			case 'product_sku':
				$product_id = $this->validateInput('product_id');
				if ($product_id) {
					return $product_id;
				}
				else {
					$product_sku = $this->validateInput('product_sku');
					if ($product_sku) {
						$q = "SELECT product_id
							FROM #__vm_product
							WHERE product_sku = ".$db->Quote($product_sku);
						$db->setQuery($q);
						$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_FIND_PRODUCT_SKU'), true);
						return $db->loadResult();
					}
					else return false;
				}
				break;
			case 'product_mpn':
				$mpn_column = $template->getValue('mpn_column_name', 'product', false);
				$product_mpn = $this->validateInput($mpn_column);
				if ($product_mpn) {
					$q = "SELECT product_id
						FROM #__vm_product
						WHERE ".$db->nameQuote($mpn_column)." = ".$db->Quote($product_mpn);
					$db->setQuery($q);
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_FIND_PRODUCT_MPN'), true);
					return $db->loadResult();
				}
				else return false;
				break;
			case 'product_child_sku':
				$product_sku = $this->validateInput('product_sku');
				$product_parent_sku = $this->validateInput('product_parent_sku');
				if ($product_sku && $product_parent_sku) {
					// Load the product parent ID
					$q = "SELECT product_id
						FROM #__vm_product
						WHERE product_sku = ".$db->Quote($product_parent_sku);
					$db->setQuery($q);
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_FIND_PRODUCT_CHILD_PARENT_SKU'), true);
					$product_parent_id = $db->loadResult();

					// Load the product ID of the child
					$q = "SELECT product_id
						FROM #__vm_product
						WHERE product_sku = ".$db->Quote($product_sku)."
						AND product_parent_id = ".$product_parent_id;
					$db->setQuery($q);
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_FIND_PRODUCT_CHILD_SKU'), true);
					return $db->loadResult();
				}
				else if ($product_sku) {
					$q = "SELECT product_id
						FROM #__vm_product
						WHERE product_sku = ".$db->Quote($product_sku);
					$db->setQuery($q);
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_FIND_PRODUCT_SKU_BASED_CHILD'), true);
					return $db->loadResult();
				}
				else {
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_NO_CHILD_NO_PARENT'));
					return false;
				}
				break;
			default:
				return false;
				break;
		}
	}

	/**
	 * Determine vendor ID
	 *
	 * Determine for which vendor we are importing product details.
	 *
	 * The default vendor is the one with the lowest vendor_id value
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo 		Add full vendor support when VirtueMart supports it
	 * @see
	 * @access 		protected
	 * @param
	 * @return 		integer	the vendor database ID
	 * @since 		3.0
	 */
	protected function getVendorId() {
		if (!$this->_vendor_id) {
			$db = JFactory::getDBO();
			$csvilog = JRequest::getVar('csvilog');

			// User is uploading vendor_id
			if (isset($this->_csvifields['vendor_id'])) {
				return $this->validateInput('vendor_id');
			}
			// User is not uploading vendor_id
			// First get the vendor with the lowest ID
			$q = "SELECT MIN(vendor_id) AS vendor_id FROM #__vm_vendor";
			$db->setQuery($q);
			$min_vendor_id = $db->loadResult();

			if (isset($this->_csvifields['product_sku'])) {
				$q = "SELECT IF (COUNT(vendor_id) = 0, ".$min_vendor_id.", vendor_id) AS vendor_id
					FROM #__vm_product
					WHERE product_sku = ".$db->Quote($this->validateInput('product_sku'));
				$db->setQuery($q);

				$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_CHECK_VENDOR_EXISTS'), true);
				// Existing vendor_id
				$vendor_id = $db->loadResult();
				$this->_vendor_id = $vendor_id;
				return $vendor_id;
			}
			// No product_sku uploaded
			else {
				$this->_vendor_id = $min_vendor_id;
				return $min_vendor_id;
			}
		}
		return $this->_vendor_id;
	}

	/**
	 * Get the shopper group id
	 *
	 * Only get the shopper group id when the shopper_group_name is set
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param		string	$shopper_group_name	the name of the shopper group to find
	 * @return 		integer	the database ID of the shopper group
	 * @since 		3.0
	 */
	protected function getShopperGroupName($shopper_group_name) {
		$db = JFactory::getDBO();
		$csvilog = JRequest::getVar('csvilog');
		$q = "SELECT shopper_group_id
			FROM #__vm_shopper_group
			WHERE shopper_group_name = ".$db->Quote($shopper_group_name);
		$db->setQuery($q);
		$this->shopper_group_id = $db->loadResult();
		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_DEBUG_SHOPPER_GROUP_NAME'), true);
		return $this->shopper_group_id;
	}

	/**
  	 * Gets the default Shopper Group ID
  	 *
  	 * @copyright
  	 * @author 		RolandD
  	 * @todo 		add error checking
  	 * @see
  	 * @access 		protected
  	 * @param
  	 * @return 		integer	the database shopper ID
  	 * @since
  	 */
	protected function getDefaultShopperGroupID() {
		$db = JFactory::getDBO();
		$csvilog = JRequest::getVar('csvilog');

		$vendor_id = $this->getVendorId();

		$q = "SELECT shopper_group_id FROM #__vm_shopper_group ";
		$q .= "WHERE `default`='1' and vendor_id='".$vendor_id."'";
		$db->setQuery($q);
		$default = $db->loadResult();
		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_DEBUG_GET_DEFAULT_SHOPPER_GROUP'), true);
		return $default;
	}

	/**
	 * Get the product type ID, cannot do without it
	 *
	 * The product_type_id is not auto incremental, therefore it needs to be
	 * set manually
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param 		string	$name	the product type name to find the ID for
	 * @return
	 * @since 		3.0
	 */
	protected function getProductTypeId($name) {
		$csvilog = JRequest::getVar('csvilog');
		$db = JFactory::getDBO();
		$q = "SELECT product_type_id
			FROM ".$db->nameQuote('#__vm_product_type')."
			WHERE product_type_name = ".$db->Quote($name);
		$db->setQuery($q);
		$id = $db->loadResult();
		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_DEBUG_PRODUCT_TYPE_ID_QUERY'), true);
		return $id;
	}

	/**
	 * Get the product currency from the vendor
	 *
	 * If the user does not use product currency we take the one from the current vendor
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		private
	 * @param 		integer	$vendor_id	the database ID of the vendor
	 * @return 		string	the 3 letter product currency
	 * @since 		3.0
	 */
	protected function productCurrency($vendor_id) {
		$csvilog = JRequest::getVar('csvilog');
		$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_DEBUG_PRODUCT_CURRENCY'));
		if (array_key_exists($vendor_id, $this->_vendor_currencies)) {
			$product_currency = strtoupper($this->_vendor_currencies[$vendor_id]);
		}
		else {
			$db = JFactory::getDBO();
			$q = "SELECT UPPER(vendor_currency) FROM #__vm_vendor WHERE vendor_id='".$vendor_id."' ";
			$db->setQuery($q);
			$product_currency = $db->loadResult();
			$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_DEBUG_PRODUCT_CURRENCY'), true);

			// Add the vendor currency to the cache
			$this->_vendor_currencies[$vendor_id] = $product_currency;
		}
		return $product_currency;
	}

	/**
	 * Validate input data
	 *
	 * Checks if the field has a value, if not check if the user wants us to
	 * use the default value
	 *
	 * @copyright
	 * @author		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param 		string $fieldname the fieldname to validate
	 * @return		true returns validated value | return false if the column count does not match
	 * @since
	 */
	protected function validateInput($fieldname) {
		$csvilog = JRequest::getVar('csvilog');
		// Check if the columns match
		if (count($this->_csvifields) > count($this->csvi_data)) {
			$message =  JText::sprintf('COM_CSVIVIRTUEMART_INCORRECT_COLUMN_COUNT', count($this->_csvifields), count($this->csvi_data));
			$message .= '<br />'.JText::_('COM_CSVIVIRTUEMART_FIELDS');
			foreach($this->_csvifields as $fieldname => $field_details) {
				$message .= '<br />'.$field_details['order'].': '.$fieldname;
			}
			$message .= '<br />'.JText::_('COM_CSVIVIRTUEMART_VALUE');
			if (!empty($this->csvi_data)) {
				foreach ($this->csvi_data AS $key => $data) {
					$message .= '<br />'.$key.': '.$data;
				}
			}
			$csvilog->AddStats('incorrect', $message, true);

			return false;
		}
		// Columns match, validate
		else {
			$template = JRequest::getVar('template');
			$newvalue = '';
			// Check if the user wants ICEcat data
			if ($template->getValue('use_icecat', 'product', false, 'bool') && !empty($this->icecat_data) && (array_key_exists($fieldname, $this->icecat_data))) {
				$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_USE_ICECAT_FIELD', $fieldname));
				$newvalue = $this->icecat_data[$fieldname];
			}
			else if (isset($this->_csvifields[$fieldname])) {
				// Check if the field has a value
				if (array_key_exists($this->_csvifields[$fieldname]["order"], $this->csvi_data)
					&& strlen($this->csvi_data[$this->_csvifields[$fieldname]["order"]]) > 0) {
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_USE_FIELD_VALUE'));
					$newvalue = trim($this->csvi_data[$this->_csvifields[$fieldname]["order"]]);
				}
				// Field has no value, check if we can use default value
				else if (!$this->_skip_default_value) {
					$csvilog->addDebug($fieldname.' :: '.JText::_('COM_CSVIVIRTUEMART_USE_DEFAULT_VALUE'). ' :: '.$this->_csvifields[$fieldname]["default_value"]);
					$newvalue = $this->_csvifields[$fieldname]["default_value"];
				}
				else {
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_USE_NO_VALUE'));
					return '';
				}
			}
			else return false;

			// Replace the value and return
			if (!empty($newvalue)) return CsviHelper::replaceValue($fieldname, $newvalue);
			else return $newvalue;
		}
	}

	/**
	 * Replace commas with periods for correct DB insertion of the prices
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo		Handle multiple separators by removing them
	 * @see
	 * @access 		protected
	 * @param 		string	$value	the value to clean up
	 * @return 		string	the cleaned up value with dots as separator
	 * @since 		3.0
	 */
	protected function toPeriod($value) {
		$clean = str_replace(",", ".", $value);
		$lastpos = strrpos($clean, '.');
		return str_replace('.', '' , substr($clean, 0, $lastpos)).substr($clean, $lastpos);
	}

	/**
	 * Format a date to unix timestamp
	 *
	 * Format of the date is day/month/year or day-month-year.
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo		use JDate
	 * @see
	 * @access 		protected
	 * @param 		string	$date	the date to convert
	 * @return		integer	UNIX timestamp if date is valid otherwise return 0
	 * @since
	 */
	protected function convertDate($date) {
		$new_date = preg_replace('/-|\./', '/', $date);
		$date_parts = explode('/', $new_date);

		if ((count($date_parts) == 3) && ($date_parts[0] > 0 && $date_parts[0] < 32 && $date_parts[1] > 0 && $date_parts[1] < 13 && (strlen($date_parts[2]) == 4))) {
			$old_date = mktime(0,0,0,$date_parts[1],$date_parts[0],$date_parts[2]);
		}
		else $old_date = 0;
		return $old_date;
	}

	/**
	 * Add the query statistics to the log
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param
	 * @return
	 * @since 		3.0
	 */
	 protected function queryResult() {
	 	$db = JFactory::getDBO();
	 	return substr($db->getQuery(), 0, strpos($db->getQuery(),' '));
	 }

	 /**
	 * Clean up a price to only exist of numbers
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param 		string	$price	the price to clean
	 * @return		float	cleaned up price
	 * @since
	 */
	protected function cleanPrice($price) {
		return JFilterInput::clean($this->toPeriod($price), 'float');
	}

	/**
	 * Load the ICEcat data for a product
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param
	 * @return
	 * @since 		3.0
	 */
	protected function getIcecat() {
		$template = JRequest::getVar('template');
		if ($template->getValue('use_icecat', 'product')) {
			$csvilog = JRequest::getVar('csvilog');

			// Load the ICEcat helper
			if (is_null($this->_icecat)) $this->_icecat = new IcecatHelper();

			// Clean the data holder
			$this->icecat_data = null;

			// Check conditions
			// 1. Do we have an MPN
			$mpn = $this->validateInput($template->getValue('mpn_column_name', 'product', 'product_sku'));
			if ($mpn) {
				$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_ICECAT_FOUND_REFERENCE', $mpn));
				// 2. Do we have a manufacturer name
				$mf_name = $this->validateInput('manufacturer_name');
				$csvilog->addDebug(JText::sprintf('COM_CSVIVIRTUEMART_ICECAT_FOUND_MF_NAME', $mf_name));
				if ($mf_name) {
					// Load the ICEcat data
					$this->icecat_data = $this->_icecat->getData($mpn, $mf_name);
				}
				else {
					$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_ICECAT_NO_MANUFACTURER'));
					return false;
				}
			}
			else {
				$csvilog->addDebug(JText::_('COM_CSVIVIRTUEMART_ICECAT_NO_REFERENCE'));
				return false;
			}
		}
		return false;

	}

	/**
	 * Set a field to combine
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param 		string	$data		the data to be combined
	 * @param 		string	$fieldname	the name of the current field
	 * @return
	 * @since 		3.0
	 */
	protected function setCombineField($data, $fieldname=null) {
		// Add the data to the array
		$this->combine_fields[] = $data;

		// Set the fieldname the data is for
		if (empty($this->combine_settings['fieldname'])) $this->combine_settings['fieldname'] = $fieldname;
		switch ($fieldname) {
			case 'category_path':
				$template = JRequest::getVar('template');
				$this->combine_settings['separator'] = $template->getValue('category_separator', 'general', '/');
				break;
			default:
				$this->combine_settings['separator'] = ' ';
				break;
		}
	}

	/**
	 * Get the combined fields
	 *
	 * @copyright
	 * @author 		RolandD
	 * @todo
	 * @see
	 * @access 		protected
	 * @param
	 * @return 		string	the space separated combined data
	 * @since 		3.0
	 */
	protected function getCombineField() {
		// Get the combined data
		$data = implode($this->combine_settings['separator'], $this->combine_fields);

		// Empty some settings
		$this->combine_fields = array();
		$this->combine_settings['fieldname'] = null;
		$this->combine_settings['separator'] = ' ';

		// Return the data
		return $data;
	}
}
?>
