package com.k_int.discover.service;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPathExpressionException;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.xpath.XPathAPI;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ApplicationEvent;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.traversal.NodeIterator;

import com.k_int.aggregator.augmentation.XMLDocumentAugmentationException;
import com.k_int.aggregator.augmentation.XMLDocumentAugmentationFactory;
import com.k_int.aggregator.augmentation.XMLDocumentAugmentor;
import com.k_int.aggregator.cache.CacheContentResult;
import com.k_int.aggregator.cache.impl.CacheContentImageThumbnail;
import com.k_int.aggregator.core.DepositResult;
import com.k_int.aggregator.dto.AggregatorCollectionDTO;
import com.k_int.aggregator.dto.AggregatorSourceDTO;
import com.k_int.aggregator.plugin.XMLDocumentHandlerPlugin;
import com.k_int.aggregator.plugin.XMLProcessingResult;
import com.k_int.aggregator.repository.RepositoryService;
import com.k_int.aggregator.repository.RepositoryStoreOptions;
import com.k_int.aggregator.repository.RepositoryStoreResult;
import com.k_int.aggregator.util.SOLRHelper;
import com.k_int.commons.util.XMLUtil;
import com.k_int.discover.datamodel.CultureGrid_CollectionDTO;
import com.k_int.discover.datamodel.CultureGrid_CollectionDocument;
import com.k_int.discover.datamodel.CultureGrid_InstitutionDTO;
import com.k_int.discover.datamodel.CultureGrid_InstitutionDocument;
import com.k_int.discover.datamodel.OAIDCDocument;
import com.k_int.discover.util.ParseSpatialData;

@Service("XMLHandler-OAI-RSLP")
public class OAIRSLPHandler implements XMLDocumentHandlerPlugin, ApplicationContextAware, org.springframework.context.ApplicationListener {

	private static Log log = LogFactory.getLog(OAIRSLPHandler.class);
	
	protected ApplicationContext ctx = null;
	
	// Namespace declarations
	public static String XMLNS_DC = "http://purl.org/dc/elements/1.1/";
	public static String XMLNS_DCTERMS = "http://purl.org/dc/terms/";
	public static String XMLNS_PNDSTERMS = "http://purl.org/mla/pnds/terms/";
	public static String XMLNS_XSI = "http://www.w3.org/2001/XMLSchema-instance";
	public static String XMLNS_PNDS_DC = "http://purl.org/mla/pnds/pndsdc/";
	public static String XMLNS_OAI_DC = "http://www.openarchives.org/OAI/2.0/oai_dc/";
	public static String XMLNS_XHTML = "http://www.w3.org/1999/xhtml";
	public static String XMLNS_OAI_RSLP = "http://www.cornucopia.org.uk/oai/oai_rslp/";
	public static String XMLNS_CLD = "http://purl.org/rslp/terms#";
	public static String XMLNS_MLA = "http://www.mla.gov.uk/schema/";
	public static String XMLNS_VCARD = "http://www.w3.org/2001/vcard-rdf/3.0#";
	public static String XMLNS_DCQ = "http://purl.org/dc/qualifiers/1.0/";
	public static String XMLNS_DCMITYPE = "http://purl.org/dc/dcmitype/";


	private static String[] SUPPORTED_SCHEMAS = new String[] { XMLNS_OAI_RSLP };
	
	public String[] getSupportedSchemas() {
		return SUPPORTED_SCHEMAS;
	}

	/**
	 * @deprecated Replaced by {@link #process(DepositResult, byte[], Document, String, String, String, Boolean, long)}
	 */
	@Deprecated
	public XMLProcessingResult process(DepositResult deposit_result,
			byte[] content, Document d, String depositor_id, String owner_id,
			String root_namespace, Boolean authoritative) {
		return this.process(deposit_result, content, d, depositor_id, owner_id, root_namespace, authoritative, (RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL | RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM | RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) );
	}

	public XMLProcessingResult process(DepositResult deposit_result,
			byte[] content, Document d, String depositor_id, String owner_id,
			String root_namespace, Boolean authoritative, long options) {

		// Set up the return value
	    XMLProcessingResult returnValue = new XMLProcessingResult();
	    
		// Set up a version of the options with store checksums set to false as used when
		// storing all versions of the document other than the original
		long noChecksumOptions = options & ~RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM;

		// Set up namespace elements
		Element new_namespace_node = d.createElement("NSNode");
		// Set up our local namespace definitions
		new_namespace_node.setAttribute("xmlns:dc", XMLNS_DC);
		new_namespace_node.setAttribute("xmlns:dcterms", XMLNS_DCTERMS);
		new_namespace_node.setAttribute("xmlns:xsi", XMLNS_XSI);
		new_namespace_node.setAttribute("xmlns:pn", XMLNS_PNDS_DC);
		new_namespace_node.setAttribute("xmlns:pndsterms", XMLNS_PNDSTERMS);
		new_namespace_node.setAttribute("xmlns:oai_dc", XMLNS_OAI_DC);
		new_namespace_node.setAttribute("xmlns:oai_rslp", XMLNS_OAI_RSLP);
		new_namespace_node.setAttribute("xmlns:dcmitype", XMLNS_DCMITYPE);
		new_namespace_node.setAttribute("xmlns:cld", XMLNS_CLD);
		new_namespace_node.setAttribute("xmlns:dcq", XMLNS_DCQ);
		new_namespace_node.setAttribute("xmlns:mla", XMLNS_MLA);
		new_namespace_node.setAttribute("xmlns:vcard", XMLNS_VCARD);

		
		try {

			// Get all of the data from the original file
			HashMap<String,NameXPathMapping> xpathMappings = this.setupMappings();
			for(String key: xpathMappings.keySet()) {
				NameXPathMapping thisOne = xpathMappings.get(key);
				String xpath = thisOne.getXpathExpression();
				String[] newValues = null;
				newValues = getValues(d, xpath, new_namespace_node);
				//   	      	  log.debug("Just mapped xpath " + xpath + " to " + newValues.length + " values");

				if (newValues != null && newValues.length != 0 ) {
					thisOne.setValues(newValues);
					//	    	          	  log.debug("stored the new found value.. in a local object");
					xpathMappings.put(key, thisOne);
					//	    	          	  log.debug("stored the value in the mappings");
				}
			}

			String doc_identifier = xpathMappings.get("identifier").getValues()[0];
			log.debug("doc_identifier: " + doc_identifier);

			// Check that we have all of the required information
			this.checkForRequiredData(xpathMappings);

			// Work out the list of collections for the data
			List<String> collectionList = new ArrayList<String>();

			if ( owner_id.equals("Cornucopia") ) {
				// Dealing with Cornucopia
				collectionList.add("Cornucopia");
			} else {
				log.debug("Unable to determine the collection list based on the identifier of the data. Identifier: " + doc_identifier);
			}
			
			// Set up the objects needed to store the different documents
			RepositoryService content_store = (RepositoryService) ctx.getBean("ContentStore");
			RepositoryStoreResult stored_resource_info = null;
			Long resourceId = -1l;

			// Store the Original format (if specified in options)
			if ( (options & RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) == RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC ) {
				stored_resource_info = content_store.store(content,owner_id,depositor_id,doc_identifier,"oai_rslp_raw","application/xml",
						SUPPORTED_SCHEMAS[0],authoritative, collectionList.toArray(new String[collectionList.size()]), options);
			} else {
				// Not storing the original format - need to go and get information about the resource
				// that is already in the database - resource id, collections lists, etc.
				log.debug("Not storing the original document, as specified by the options");
				
				stored_resource_info = content_store.lookupResourceInfo(owner_id, doc_identifier);
				if ( stored_resource_info == null ) {
					log.error("Not storing the original document and unable to find it in the database to retrieve information about it!!");
					throw(new XMLDocumentAugmentationException("Unable to retrieve information about the original document from the database to set up data augmentation"));
				}
			}
			
			// Get the information from the result of storing the new doc or looking up the original in the DB to
			// control augmentation below.
			resourceId = stored_resource_info.getDocId();
			AggregatorSourceDTO sourceDetails = stored_resource_info.getSource();
			List<AggregatorCollectionDTO> collectionDetails = stored_resource_info.getCollections();

			returnValue.setDocId(resourceId);
			
			// Set up the augmentation factory and get the relevant augmentor for this type
			// of object
			XMLDocumentAugmentationFactory augFactory = (XMLDocumentAugmentationFactory)ctx.getBean("AugmentationFactory");
			XMLDocumentAugmentor augmentor = (XMLDocumentAugmentor)augFactory.getAugmentor("COLLECTION");
			
			
			if ( augmentor == null ) { 
				log.error("augmentor is null - not able to augment the data");
			} else {
				log.debug("non-null augmentor - ready to start augmenting data.");
				
				// Set up the DTO to be passed to the augmentor
				CultureGrid_CollectionDTO collDTO = new CultureGrid_CollectionDTO();
				if ( checkHasValueToStore(xpathMappings, "accessConditions") ) { 
					collDTO.setAccessConditions(xpathMappings.get("accessConditions").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "addressPart") ) { 
					collDTO.setAddressPart(xpathMappings.get("addressPart").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "audience") ) {
					List<String> audienceList = convertArrayToList(xpathMappings.get("audience").getValues());
					collDTO.setAudience(audienceList);
				}
				if ( checkHasValueToStore(xpathMappings, "description") ) {
					// Combine the descriptions together into one and then store it
					StringBuilder completeDescription = new StringBuilder();
					for(String thisDesc: xpathMappings.get("description").getValues() ) {
						if ( thisDesc != null && !"".equals(thisDesc.trim())) {
							if ( completeDescription.length() > 0 )
								completeDescription.append("\n");
							completeDescription.append(thisDesc.trim());
						}
					}
					
					if ( completeDescription.length() > 0 ) {
						collDTO.setDescription(completeDescription.toString());
					}
				}
				if ( checkHasValueToStore(xpathMappings, "disabilityAccess") ) {
					collDTO.setDisabledAccess(xpathMappings.get("disabilityAccess").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "eMail") ) {
					collDTO.setEmail(xpathMappings.get("eMail").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "faxNumber") ) {
					collDTO.setFax(xpathMappings.get("faxNumber").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "format") ) {
					collDTO.setFormat(xpathMappings.get("format").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "identifier") ) {
					collDTO.setIdentifier(xpathMappings.get("identifier").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "language") ) {
					collDTO.setLanguage(xpathMappings.get("language").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "license") ) {
					collDTO.setLicense(xpathMappings.get("license").getValues()[0]); 
				}
				if ( checkHasValueToStore(xpathMappings, "license_text") ) {
					collDTO.setLicenseText(xpathMappings.get("license_text").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "operational_hours") ) {
					collDTO.setOperationalHours(xpathMappings.get("operational_hours").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "otherIdentifier") && checkHasValueToStore(xpathMappings, "otherIdentifierScheme") ) {
					String[] otherIdentifiers = xpathMappings.get("otherIdentifier").getValues();
					String[] otherIdentifierSchemes = xpathMappings.get("otherIdentifierScheme").getValues();
					if ( otherIdentifiers.length != otherIdentifierSchemes.length ) {
						log.error("Collected a different number of other identifiers (" + otherIdentifiers.length + ") to other identifier schemes (" + otherIdentifierSchemes.length + ") from the source document.");
					} else {
						// Matching numbers of other identifiers and their schemes - remember them in the DTO
						HashMap<String,String> otherIdentifierInfo = new HashMap<String,String>();
						for(int ctr = 0; ctr < otherIdentifiers.length; ctr++) {
							otherIdentifierInfo.put(otherIdentifierSchemes[ctr], otherIdentifiers[ctr]);
						}
						
						collDTO.setOtherIdentifiers(otherIdentifierInfo);
					}
				}
				if ( checkHasValueToStore(xpathMappings, "owner") ) {
					collDTO.setOwner(xpathMappings.get("owner").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "phoneNumber") ) {
					collDTO.setPhone(xpathMappings.get("phoneNumber").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "postcode") ) {
					collDTO.setPostcode(xpathMappings.get("postcode").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "rights") ) {
					collDTO.setRights(xpathMappings.get("rights").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "rightsHolder") ) {
					collDTO.setRightsHolder(xpathMappings.get("rightsHolder").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "seeAlso") ) {
					List<String> seeAlsoList = convertArrayToList(xpathMappings.get("seeAlso").getValues());
					collDTO.setSeeAlso(seeAlsoList);
				}
				if ( checkHasValueToStore(xpathMappings, "source") ) {
					collDTO.setSource(xpathMappings.get("source").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "spatial") ) {
					List<String> spatialList = convertArrayToList(xpathMappings.get("spatial").getValues());
					collDTO.setSpatial(spatialList);
				}
				if ( checkHasValueToStore(xpathMappings, "subject") ) {
					List<String> subjectList = convertArrayToList(xpathMappings.get("subject").getValues());
					collDTO.setSubject(subjectList);
				}
				if ( checkHasValueToStore(xpathMappings, "temporal") ) {
					List<String> temporalList = convertArrayToList(xpathMappings.get("temporal").getValues());
					collDTO.setTemporal(temporalList);
				}
				if ( checkHasValueToStore(xpathMappings, "thumbnail") ) {
					collDTO.setThumbnail(xpathMappings.get("thumbnail").getValues()[0]);
				}
				StringBuilder completeTitle = new StringBuilder();
				if ( checkHasValueToStore(xpathMappings, "collectionTitle") && xpathMappings.get("collectionTitle").getValues()[0] != null ) {
					completeTitle.append(xpathMappings.get("collectionTitle").getValues()[0]);
				}
				if ( checkHasValueToStore(xpathMappings, "locationTitle") && xpathMappings.get("locationTitle").getValues()[0] != null ) {
					if ( completeTitle.length() > 0 ) {
						completeTitle.append(" - ");
					}
					completeTitle.append(xpathMappings.get("locationTitle").getValues()[0]);
				}
				if ( completeTitle.length() > 0 ) {
					collDTO.setTitle(completeTitle.toString());
				}
				if ( checkHasValueToStore(xpathMappings, "usageConditions") ) {
					collDTO.setUsageConditions(xpathMappings.get("usageConditions").getValues()[0]);
				}
				
				// Perform the augmentation getting back the list of generated objects to be stored / indexed
				HashMap<String, Object> augmentedData = augmentor.augmentData(collDTO, depositor_id, sourceDetails, root_namespace, collectionDetails, resourceId, options);

				// Loop through the returned augmented data, storing each of the different documents as appropriate
				Set<String> augmentedKeyset = augmentedData.keySet();
				Iterator<String> augmentedIterator = augmentedKeyset.iterator();
				
				while(augmentedIterator.hasNext()) {
					String type = augmentedIterator.next();
					Object doc = augmentedData.get(type);

					String mime_type = "";
					String sub_type = "";
					byte[] contents = null;
					
					log.debug("OAIRSLP handler dealing with the augmented document of type: " + type);
					if ( type.equals("oai_dc") ) {
						// OAI_DC document
						mime_type = "application/xml";
						sub_type = "http://www.openarchives.org/OAI/2.0/oai_dc/";
						OAIDCDocument oaiDcDoc = (OAIDCDocument)doc;
						contents = XMLUtil.serializeDocument(oaiDcDoc.toXML());
					} else if ( type.equals("CultureGrid_Collection") ) {
						// CultureGrid internal collection format
						mime_type = "application/xml";
						sub_type = "http://www.peoplesnetwork.gov.uk/schema/CultureGrid_Collection"; // TODO - what should this really be?
						CultureGrid_CollectionDocument cgCollDoc = (CultureGrid_CollectionDocument)doc;
						contents = XMLUtil.serializeDocument(cgCollDoc.toXML());
					} else if ( type.equals("thumbnail_image_jpeg") ) {
						// Thumbnail image
						mime_type = "image/jpeg";
						sub_type = "";
						CacheContentResult cachedThumb = (CacheContentResult)doc;
						contents = cachedThumb.content;
					} else if ( type.equals("solr") ) {
						// Solr index properties
						mime_type = "application/xml";
						sub_type = "";
						List<SOLRHelper.NVPair> index_properties = (List<SOLRHelper.NVPair>)doc;
						
						// Solr is different - we need to do the indexing here - pass the returned properties through
						// to SOLR and then store the returned XML document
						
						SOLRHelper solr_helper = (com.k_int.aggregator.util.SOLRHelper) ctx.getBean("SolrHelper");
				
						log.debug("Indexing the document into solr");
						Document solr_doc = solr_helper.postIndexEntries(index_properties, false);
						if ( solr_doc != null ) {
							contents = XMLUtil.serializeDocument(solr_doc);
						} else {
							log.debug("An error occurred when indexing the document to solr - no document returned! doc_identifier: " + doc_identifier);
						}
					} else {
						// Unrecognised - don't know what to do, so log it..
						log.error("Unrecognised type of document returned from augmenting collection data. Type: " + type + ". Object class: " + doc.getClass().getName());
					}
					
					// The document is now processed and we have all of the information we need to store it - do so
					if ( contents != null ) {
						// We have something to store - do so
						log.debug("Storing a document of type: " + type + ", mime_type: " + mime_type + ", sub_type: " + sub_type);
						content_store.store(contents, owner_id, depositor_id, doc_identifier, type, mime_type, sub_type, authoritative, 
								collectionList.toArray(new String[collectionList.size()]), noChecksumOptions);
					}
					
				}

			}

			
				
	        returnValue.setDocId(resourceId);

			
		} catch (TransformerException te) {
			log.error("Problem - transformer exception: " + te);
			returnValue.setMessage("TransformerException thrown when parsing the OAI_RSLP document");
		} catch (XPathExpressionException xe) {
			log.error("Problem when parsing the OAI_RSLP document. " + xe.getMessage());
			returnValue.setMessage(xe.getMessage());
		} catch (XMLDocumentAugmentationException ae) {
			log.error("Problem - XMLDocumentAugmentationException: " + ae); // TODO - should i handle this differently?
			returnValue.setMessage(ae.getMessage());
		}

        return returnValue;
	}

	public void onApplicationEvent(ApplicationEvent evt) {
		log.info("Spring event : "+evt);
	}

	public void setApplicationContext(ApplicationContext ctx)
			throws BeansException {
		this.ctx = ctx;
	}
	
	private HashMap<String,NameXPathMapping> setupMappings() {

		HashMap<String, NameXPathMapping> mappings = new HashMap<String,NameXPathMapping>();

		  mappings.put("accessConditions", new NameXPathMapping("cld.accessConditions", "/oai_rslp:rslp/cld:Location/cld:accessConditions", false)); 
//		  mappings.put("accessControl", new NameXPathMapping("cld.accessControl", "/oai_rslp:rslp/dcmitype:Collection/cld:accessControl", false)); 
//		  mappings.put("accrualStatus", new NameXPathMapping("cld.accrualStatus", "/oai_rslp:rslp/dcmitype:Collection/cld:accrualStatus", false)); 
//		  mappings.put("accumulationDateRange", new NameXPathMapping("cld.accumulationDateRange", "/oai_rslp:rslp/dcmitype:Collection/cld:accumulationDateRange", false));
		  mappings.put("addressPart", new NameXPathMapping("NO_INDEX", "/oai_rslp:rslp/cld:Location/cld:address", false));
//		  mappings.put("administratorID", new NameXPathMapping("cld.Administrator", "/oai_rslp:rslp/cld:Administrator/@about", false)); 
//		  mappings.put("administratorEMail", new NameXPathMapping("vcard.email", "/oai_rslp:rslp/cld:Administrator/vcard:email", false));
//		  mappings.put("administratorName", new NameXPathMapping("vcard.fn", "/oai_rslp:rslp/cld:Administrator/vcard:fn", false)); 
//		  mappings.put("administratorOrganisation", new NameXPathMapping("vcard.org", "/oai_rslp:rslp/cld:Administrator/vcard:org", false));
//		  mappings.put("administrativeStatus", new NameXPathMapping("mla.administrative_status", "/oai_rslp:rslp/cld:Location/mla:administrative_status", false));
//		  mappings.put("alternativeName", new NameXPathMapping("mla.alternative_name", "/oai_rslp:rslp/cld:Location/mla:alternative_name", false)); 
		  mappings.put("audience", new NameXPathMapping("dcterms.audience","/oai_rslp:rslp/dcmitype:Collection/dcterms:audience", false));
//		  mappings.put("contributor", new NameXPathMapping("dc.contributor","/pn:description/dc:contributor", false));
//		  mappings.put("contentsDateRange", new NameXPathMapping("cld.contentsDateRange", "/oai_rslp:rslp/dcmitype:Collection/cld:contentsDateRange", false));
//		  mappings.put("custodialHistory", new NameXPathMapping("cld.custodialHistory", "/oai_rslp:rslp/dcmitype:Collection/cld:custodialHistory", false)); 
		  mappings.put("description", new NameXPathMapping("dc.description","/oai_rslp:rslp/dcmitype:Collection/dc:description", false, true));
		  mappings.put("disabilityAccess", new NameXPathMapping("disabledAccess", "/oai_rslp:rslp/cld:Location/mla:disability_access", false)); 
		  mappings.put("eMail", new NameXPathMapping("vcard.email", "/oai_rslp:rslp/cld:Location/vcard:email", false));
		  mappings.put("faxNumber", new NameXPathMapping("vcard.fax", "/oai_rslp:rslp/cld:Location/vcard:fax", false)); 
		  mappings.put("format", new NameXPathMapping("dc.format","/oai_rslp:rslp/dcmitype:Collection/dc:format", false));
		  mappings.put("identifier", new NameXPathMapping("dc.identifier","/oai_rslp:rslp/cld:Location/cld:isLocationOf", true));
//		  mappings.put("institutionHistory", new NameXPathMapping("mla.institution_history", "/oai_rslp:rslp/cld:Location/mla:institution_history", false));
//		  mappings.put("jurisdiction", new NameXPathMapping("mla.jurisdiction","/oai_rslp:rslp/cld:Location/mla:jurisdiction", false)); 
		  mappings.put("language", new NameXPathMapping("dc.language","/oai_rslp:rslp/dcmitype:Collection/dcterms:language", false));
//		  mappings.put("legalStatus", new NameXPathMapping("cld.legalStatus", "/oai_rslp:rslp/dcmitype:Collection/cld:legalStatus", false));
		  mappings.put("license", new NameXPathMapping("dcterms.license","/oai_rslp:rslp/dcmitype:Collection/dcterms:license/@valueURI", false));
		  mappings.put("license_text", new NameXPathMapping("NO_INDEX", "/oai_rslp:rslp/dcmitype:Collection/dcterms:license", false));
//		  mappings.put("location", new NameXPathMapping("dc.location","/pn:description/dc:location", false));
//		  mappings.put("logo", new NameXPathMapping("mla.logo", "/oai_rslp:rslp/cld:Location/mla:logo", false)); 
//		  mappings.put("mlaRegion", new NameXPathMapping("mla.mla_region", "/oai_rslp:rslp/cld:Location/mla:mla_region", false));
//		  mappings.put("note", new NameXPathMapping("cld.note", "/oai_rslp:rslp/dcmitype:Collection/cld:note", false)); 
		  mappings.put("operational_hours", new NameXPathMapping("mla.operational_hours", "/oai_rslp:rslp/cld:Location/mla:operational_hours", false));
		  mappings.put("otherIdentifier", new NameXPathMapping("NO_INDEX", "/oai_rslp:rslp/cld:Location/mla:other_identifier", false, true));
		  mappings.put("otherIdentifierScheme", new NameXPathMapping("NO_INDEX", "/oai_rslp:rslp/cld:Location/mla:other_identifier/@scheme", false, true));
		  mappings.put("owner", new NameXPathMapping("cld.owner", "/oai_rslp:rslp/cld:Owner/vcard:fn", false));
		  mappings.put("phoneNumber", new NameXPathMapping("vcard.voice", "/oai_rslp:rslp/cld:Location/vcard:voice", false)); 
		  mappings.put("place", new NameXPathMapping("dcterms.spatial","/oai_rslp:rslp/cld:Location/cld:postcode", false));
		  mappings.put("postcode", new NameXPathMapping("unparsed.postcode","/oai_rslp:rslp/cld:Location/cld:postcode", false));
//		  mappings.put("publisher", new NameXPathMapping("dc.publisher","/pn:description/dc:publisher", false));
//		  mappings.put("referral", new NameXPathMapping("mla.referral", "/oai_rslp:rslp/cld:Location/mla:referral", false)); 
//		  mappings.put("regionCounty", new NameXPathMapping("mla.region_county", "/oai_rslp:rslp/cld:Location/mla:region_county", false)); 
		  mappings.put("rights", new NameXPathMapping("dc.rights","/oai_rslp:rslp/dcmitype:Collection/dc:rights", false));
		  mappings.put("rightsHolder", new NameXPathMapping("dcterms.rightsHolder","/oai_rslp:rslp/dcmitype:Collection/dcterms:rightsHolder", false));
//		  mappings.put("sector", new NameXPathMapping("mla.sector", "/oai_rslp:rslp/cld:Location/mla:sector", false)); 
		  mappings.put("seeAlso", new NameXPathMapping("cld.seeAlso", "/oai_rslp:rslp/cld:Location/cld:seeAlso", false));
		  mappings.put("source", new NameXPathMapping("dc.source", "/oai_rslp:rslp/dcmitype:Collection/dc:source", false));
		  mappings.put("spatial", new NameXPathMapping("dcterms.spatial", "/oai_rslp:rslp/dcmitype:Collection/dcterms.spatial", false, true));
		  mappings.put("subject", new NameXPathMapping("dc.subject","/oai_rslp:rslp/dcmitype:Collection/dc:subject", false, true));
		  mappings.put("temporal", new NameXPathMapping("dcterms.temporal", "/oai_rslp:rslp/dcmitype:Collection/dcterms:temporal", false, true));
		  mappings.put("thumbnail", new NameXPathMapping("pndsterms.thumbnail","/oai_rslp:rslp/dcmitype:Collection/pndsterms:thumbnail/@valueURI", false));
		  mappings.put("collectionTitle", new NameXPathMapping("NO_INDEX","/oai_rslp:rslp/dcmitype:Collection/dc:title", false));
		  mappings.put("locationTitle", new NameXPathMapping("NO_INDEX", "/oai_rslp:rslp/cld:Location/dc:title", false));
//		  mappings.put("type", new NameXPathMapping("dcmi.type", "/oai_rslp:rslp/dcmitype:Collection/dc:type", false)); // TODO - what should this get mapped to???
		  mappings.put("usageConditions", new NameXPathMapping("mla.usage_conditions", "/oai_rslp:rslp/cld:Location/mla:usage_conditions", false));

		return mappings;
	}


	/**
	 * Check through the data in the mappings (once filled with values) - if something is
	 * required then check that it isn't null i.e. that we have found a value for it
	 * @throws XPathExpressionException thrown if a field that is required has not been filled in from the XML document
	 */
	private void checkForRequiredData(HashMap<String,NameXPathMapping> xpathMappings) throws XPathExpressionException {
		// Loop through all of the entries and consider those where required == true
		for(String key: xpathMappings.keySet()) {
			NameXPathMapping thisEntry = xpathMappings.get(key);
			if ( thisEntry.isRequired() ) {
				// We care about it being a non-null value - check
				String[] value = thisEntry.getValues();
				if ( value == null ) {
					log.debug("The required field: " + key + " doesn't have a value in the data file");
					throw(new XPathExpressionException("Required field: " + key + " does not have a value in the data"));
				}
			}
		}
	}
	  
	/**
	 * Get the value of the element or attribute given by the specified xpath expression
	 * @param metadata_record The document to apply the xpath to
	 * @param xpath The xpath expression to the element or attribute to consider
	 * @param namespace_node The set of namespaces used in the xpath and document
	 * @return The text contents of the elements or attributes as an array
	 * @throws javax.xml.transform.TransformerException thrown if there's an error with the XPath
	 */
	protected static String[] getValues(Node metadata_record, String xpath, Node namespace_node) throws javax.xml.transform.TransformerException {
		// First work out whether we're looking for an attribute or a full element
		// - we're looking at an attribute if there is an @ after the last / 
		// (and the expression doesn't end with ].
		String node_xpath = xpath;
		String attribute_xpath = "";

		if ( !xpath.endsWith("]") ) {

			int indexOfLastSlash = xpath.lastIndexOf("/");
			int indexOfLastAt = xpath.lastIndexOf("@");

			if ( indexOfLastAt > indexOfLastSlash ) {
				// Dealing with an attribute
				attribute_xpath = xpath.substring(indexOfLastAt + 1);
				node_xpath = xpath.substring(0, indexOfLastAt - 1);
			}
		}

		// Go and get the node(s) we're interested in
		NodeIterator nodeList = XPathAPI.selectNodeIterator(metadata_record, node_xpath, namespace_node);
		String[] returnValue = null;
		ArrayList<String> tempList = new ArrayList<String>();

		Node actualNode;
		while ((actualNode = nodeList.nextNode()) != null) {
			// We've got a node - now are we getting an attribute or not?
			String value = null;
			if ( !"".equals(attribute_xpath) ) {
				value = ((Element)actualNode).getAttribute(attribute_xpath);
			} else {
				value = extractText(actualNode);
			}

			// Unescape any html entites, etc. in the string (a couple of times since there are some places
			// that encode multiple times and then remove the HTML
			//			  log.debug("about to unescape the value: " + value);
			if ( value != null ) {
				value = StringEscapeUtils.unescapeHtml(StringEscapeUtils.unescapeHtml(value));
				value = value.replaceAll("\\<.*?\\>", "");
			}

			//			  log.debug("value after unescaping and stripping tags: " + value);

			tempList.add(value);
		}

		returnValue = (String[])tempList.toArray(new String[tempList.size()]);

		return returnValue;
	}


	protected static String extractText(Node n) {
		try {
			Node node = XPathAPI.selectSingleNode(n,"./text()");
			if ( node != null ) {
				node.normalize();
				return node.getNodeValue().trim();
			}
		}
		catch ( javax.xml.transform.TransformerException te ) {
			te.printStackTrace();
		}
		return null;
	}
	
	private boolean checkHasValueToStore(HashMap<String,NameXPathMapping> mappings, String key) {
		if ( mappings.containsKey(key) && mappings.get(key).getValues() != null && mappings.get(key).getValues().length > 0 ) {
			return true;
		} else {
			return false;
		}
	}

	private List<String> convertArrayToList(String[] valueArray) {
		List<String> valueList = null;
		if ( valueArray != null && valueArray.length > 0 ) {
			valueList = new ArrayList<String>();
			for(String value: valueArray) {
				if ( value != null && !"".equals(value.trim()) ) {
					valueList.add(value.trim());
				}
			}
		}
		if ( valueList != null && valueList.size() == 0 ) {
			valueList = null;
		}
		
		return valueList;
	}
}