package com.k_int.discover.service;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;

import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPathExpressionException;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.xpath.XPathAPI;
import org.hibernate.HibernateException;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ApplicationEvent;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.traversal.NodeIterator;

import com.k_int.aggregator.cache.CacheContentResult;
import com.k_int.aggregator.cache.impl.CacheContentImageThumbnail;
import com.k_int.aggregator.core.DepositResult;
import com.k_int.aggregator.plugin.XMLDocumentHandlerPlugin;
import com.k_int.aggregator.plugin.XMLProcessingResult;
import com.k_int.aggregator.repository.RepositoryService;
import com.k_int.aggregator.repository.RepositoryStoreOptions;
import com.k_int.aggregator.repository.RepositoryStoreResult;
import com.k_int.aggregator.util.SOLRHelper;
import com.k_int.commons.util.XMLUtil;
import com.k_int.discover.datamodel.OAIDCDocument;
import com.k_int.discover.datamodel.CultureGrid_ItemDocument;
import com.k_int.discover.util.ParseSpatialData;

@Service("XMLHandler-Michael")
public class MichaelHandler implements XMLDocumentHandlerPlugin, ApplicationContextAware, org.springframework.context.ApplicationListener {

	private static Log log = LogFactory.getLog(MichaelHandler.class);
	
	protected ApplicationContext ctx = null;
	
	// Namespace declarations
	public static String XMLNS_DC = "http://purl.org/dc/elements/1.1/";
	public static String XMLNS_DCTERMS = "http://purl.org/dc/terms/";
	public static String XMLNS_PNDSTERMS = "http://purl.org/mla/pnds/terms/";
	public static String XMLNS_XSI = "http://www.w3.org/2001/XMLSchema-instance";
	public static String XMLNS_PNDS_DC = "http://purl.org/mla/pnds/pndsdc/";
	public static String XMLNS_OAI_DC = "http://www.openarchives.org/OAI/2.0/oai_dc/";
	public static String XMLNS_XHTML = "http://www.w3.org/1999/xhtml";
	public static String XMLNS_OAI_COMPLETE = "http://michael-culture.org/namespaces/michael/oai/complete/";
	public static String XMLNS_MICHAEL = "http://michael-culture.org/namespaces/michael";

	private static String[] SUPPORTED_SCHEMAS = new String[] { XMLNS_OAI_COMPLETE };
	
	public String[] getSupportedSchemas() {
		return SUPPORTED_SCHEMAS;
	}

	/**
	 * @deprecated Replaced by {@link #process(DepositResult, byte[], Document, String, String, String, Boolean, long)}
	 */
	@Deprecated
	public XMLProcessingResult process(DepositResult deposit_result,
			byte[] content, Document d, String depositor_id, String owner_id,
			String root_namespace, Boolean authoritative) {
		return this.process(deposit_result, content, d, depositor_id, owner_id, root_namespace, authoritative, (RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL | RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM | RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) );
	}

	public XMLProcessingResult process(DepositResult deposit_result,
			byte[] content, Document d, String depositor_id, String owner_id,
			String root_namespace, Boolean authoritative, long options) {

		// Set up the return value
	    XMLProcessingResult returnValue = new XMLProcessingResult();
	    
		// Set up a version of the options with store checksums set to false as used when
		// storing all versions of the document other than the original
		long noChecksumOptions = options & ~RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM;

		// Set up namespace elements
		Element new_namespace_node = d.createElement("NSNode");
		// Set up our local namespace definitions
		new_namespace_node.setAttribute("xmlns:dc", XMLNS_DC);
		new_namespace_node.setAttribute("xmlns:dcterms", XMLNS_DCTERMS);
		new_namespace_node.setAttribute("xmlns:xsi", XMLNS_XSI);
		new_namespace_node.setAttribute("xmlns:pn", XMLNS_PNDS_DC);
		new_namespace_node.setAttribute("xmlns:pndsterms", XMLNS_PNDSTERMS);
		new_namespace_node.setAttribute("xmlns:oai_dc", XMLNS_OAI_DC);
		new_namespace_node.setAttribute("xmlns:oai_complete", XMLNS_OAI_COMPLETE);
		new_namespace_node.setAttribute("xmlns:michael", XMLNS_MICHAEL);
		
		try {

			// Get all of the data from the original file
			HashMap<String,NameXPathMapping> xpathMappings = this.setupMappings();
			for(String key: xpathMappings.keySet()) {
				NameXPathMapping thisOne = xpathMappings.get(key);
				String xpath = thisOne.getXpathExpression();
				String[] newValues = null;
				newValues = getValues(d, xpath, new_namespace_node);
				//   	      	  log.debug("Just mapped xpath " + xpath + " to " + newValues.length + " values");

				if (newValues != null && newValues.length != 0 ) {
					thisOne.setValues(newValues);
					//	    	          	  log.debug("stored the new found value.. in a local object");
					xpathMappings.put(key, thisOne);
					//	    	          	  log.debug("stored the value in the mappings");
				}
			}

			// Perform some modifications for the data as required by the different providers to 
			// make their data correct

			xpathMappings = this.normaliseMichaelDataFormats(xpathMappings);

			String doc_identifier = xpathMappings.get("identifier").getValues()[0];
			log.debug("doc_identifier: " + doc_identifier);

			// Modify and / or normalise the data as appropriate
			xpathMappings = this.normaliseData(xpathMappings);


			// Check that we have all of the required information
			this.checkForRequiredData(xpathMappings);

			// Work out whether we have a link to the resource and remember it
			if ( doc_identifier.startsWith("http://") || doc_identifier.startsWith("https://") ) {
				String[] relatedLink = {doc_identifier};
				NameXPathMapping relatedLinkEntry = new NameXPathMapping("dc.related.link","", false);
				relatedLinkEntry.setValues(relatedLink);
				
				xpathMappings.put("relatedLink", relatedLinkEntry);
			}
			
			// Work out the list of collections for the data
			List<String> collectionList = new ArrayList<String>();

			if (doc_identifier.startsWith("UK-DC")) {
				// Dealing with Michael DCollections
				collectionList.add("MichaelUKDCollections");
			} else if (doc_identifier.startsWith("sdx:www.michael-culture.org.uk:80:pub-uk/project/")) {
				// Michael UK Projects
				collectionList.add("MichaelUKProject");
			} else if (doc_identifier.startsWith("IS:")) {
				// Michael UK Institutions
				collectionList.add("MichaelUKInstitutions");
			} else if (doc_identifier.startsWith("UK-SE-")) {
				// Michael UK Service
				collectionList.add("MichaelUKService");
			} else {
				log.debug("Unable to determine the collection list based on the identifier of the data. Identifier: " + doc_identifier);
			}

			// Set up the objects needed to store the different documents
			RepositoryService content_store = (RepositoryService) ctx.getBean("ContentStore");
			Long resourceId = -1l;

			// Store the Original format (if specified in options)
			if ( (options & RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) == RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC ) {
				RepositoryStoreResult store_result = content_store.store(content,owner_id,depositor_id,doc_identifier,"oai_complete_raw","application/xml",
						SUPPORTED_SCHEMAS[0],authoritative, collectionList.toArray(new String[collectionList.size()]), options);
				resourceId = store_result.getDocId();
			} else {
				log.debug("Not storing the original document, as specified by the options");
			}

			// Create an OAI_DC representation
			OAIDCDocument oaiDcDoc = new OAIDCDocument();
			if ( xpathMappings.containsKey("description") && xpathMappings.get("description").getValues() != null && xpathMappings.get("description").getValues().length > 0 ) {
				oaiDcDoc.setDescription(xpathMappings.get("description").getValues()[0]);
			}
			if ( xpathMappings.containsKey("identifier") && xpathMappings.get("identifier").getValues() != null && xpathMappings.get("identifier").getValues().length > 0 ) {
				oaiDcDoc.setIdentifier(xpathMappings.get("identifier").getValues()[0]);
			}
			if ( xpathMappings.containsKey("title") && xpathMappings.get("title").getValues() != null && xpathMappings.get("title").getValues().length > 0 ) {
				oaiDcDoc.setTitle(xpathMappings.get("title").getValues()[0]);
			}
			oaiDcDoc.setType("Collection");
			oaiDcDoc.setCreator(owner_id);

			// Store the OAI_DC version
			log.debug("Storing the generated oai_dc document in the content store");
			byte[] serialisedOaiDc = XMLUtil.serializeDocument(oaiDcDoc.toXML());
			if ( serialisedOaiDc != null ) {
				RepositoryStoreResult store_result = content_store.store(serialisedOaiDc,owner_id, depositor_id, doc_identifier, "oai_dc", "application/xml", "http://www.openarchives.org/OAI/2.0/oai_dc/", authoritative,
						collectionList.toArray(new String[collectionList.size()]), noChecksumOptions);

				if ( resourceId == -1l ) {
					// We didn't have a resource ID before, remember it now
					resourceId = store_result.getDocId();
				}
			}

			// Create a PNDS_DCAP representation
			// TODO - what instead of PNDS_DCAP?
//			PNDS_DCAPDocument pndsdcapDoc = new PNDS_DCAPDocument();
//			if ( xpathMappings.containsKey("identifier") && xpathMappings.get("identifier").getValues() != null && xpathMappings.get("identifier").getValues().length > 0 ) {
//				pndsdcapDoc.setIdentifier(xpathMappings.get("identifier").getValues()[0]);
//			}
//			if ( xpathMappings.containsKey("title") &&  xpathMappings.get("title").getValues() != null && xpathMappings.get("title").getValues().length > 0 ) {
//				pndsdcapDoc.setTitle(xpathMappings.get("title").getValues()[0]);
//			}
//			if ( xpathMappings.containsKey("description") && xpathMappings.get("description").getValues() != null && xpathMappings.get("description").getValues().length > 0 ) {
//				pndsdcapDoc.setDescription(xpathMappings.get("description").getValues()[0]);
//			}
//			if ( xpathMappings.containsKey("subject") && xpathMappings.get("subject").getValues() != null && xpathMappings.get("subject").getValues().length > 0 ) {
//				List<String> subjects = new ArrayList<String>();
//				String[] tempSubjectArray = xpathMappings.get("subject").getValues();
//				for(String thisSubject: tempSubjectArray ) {
//					subjects.add(thisSubject);
//				}
//				pndsdcapDoc.setSubject(subjects);
//			}
//			if ( xpathMappings.containsKey("type") && xpathMappings.get("type").getValues() != null && xpathMappings.get("type").getValues().length > 0 ) {
//				List<String> types = new ArrayList<String>();
//				String[] tempTypesArray = xpathMappings.get("type").getValues();
//				for(String thisType: tempTypesArray ) {
//					types.add(thisType);
//				}
//				pndsdcapDoc.setType(types);
//			}
//			if ( xpathMappings.containsKey("spatial") && xpathMappings.get("spatial").getValues() != null && xpathMappings.get("spatial").getValues().length > 0 ) {
//				pndsdcapDoc.setSpatial(xpathMappings.get("spatial").getValues()[0]);
//			}
//			if ( xpathMappings.containsKey("temporal") && xpathMappings.get("temporal").getValues() != null && xpathMappings.get("temporal").getValues().length > 0 ) {
//				pndsdcapDoc.setTemporal(xpathMappings.get("temporal").getValues()[0]);
//			}
//			if ( xpathMappings.containsKey("thumbnail") && xpathMappings.get("thumbnail").getValues() != null && xpathMappings.get("thumbnail").getValues().length > 0 ) {
//				pndsdcapDoc.setThumbnail(xpathMappings.get("thumbnail").getValues()[0]);
//			}
//			if ( collectionList != null ) {
//				pndsdcapDoc.setIsPartOf(collectionList);
//			}
//
//			// Store the PNDS_DCAP version
//			log.debug("Storing the generated pnds_dcap document in the content store");
//			byte[] serialisedPndsDcapDoc = XMLUtil.serializeDocument(pndsdcapDoc.toXML());
//			if ( serialisedPndsDcapDoc != null ) {
//				content_store.store(serialisedPndsDcapDoc,owner_id, depositor_id, doc_identifier, "pnds_dcap", "application/xml", "http://purl.org/mla/pnds/pndsdc/", authoritative,
//						collectionList.toArray(new String[collectionList.size()]), noChecksumOptions);
//			}

			// Download and create a thumbnail, storing it in the db if appropriate (and if specified in options)
			NameXPathMapping thumbnailEntry = xpathMappings.get("thumbnail");
			String haveThumbnail = "true";
			String possiblePathToCachedImage = "/dpp/resource/" + resourceId + "/stream/thumbnail_image_jpeg";
			if ( thumbnailEntry != null && thumbnailEntry.getValues() != null && thumbnailEntry.getValues()[0] != null ) {

				// Go and download the thumbnail resizing it as appropriate and then store it into the database (if the options
				// specify that we should)

				if ( (options & RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL) == RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL ) {
					CacheContentResult cachedThumbnail = null;
					CacheContentImageThumbnail resizer = new CacheContentImageThumbnail();
					cachedThumbnail = resizer.cacheObject(thumbnailEntry.getValues()[0]);

					if ( cachedThumbnail != null ) {
						// Store the thumbnail in the database and the path to it in the solr
						content_store.store(cachedThumbnail.content,owner_id, depositor_id, doc_identifier, cachedThumbnail.type,
								cachedThumbnail.mimeType, cachedThumbnail.subType, authoritative,collectionList.toArray(new String[collectionList.size()]), noChecksumOptions);
					} else {
						haveThumbnail = "false";
					}
				} else {
					// Look in the database to see if there's already a thumbnail for this record
					// and if there is use it instead
					Long thumbnailID = null;
					thumbnailID = content_store.lookupResourceInstanceId(resourceId, "thumbnail_image_jpeg");
					if ( thumbnailID == null ) {
						haveThumbnail = "false";
					} else {
						log.debug("Using previously cached thumbnail");
					}
				}

			} else {
				haveThumbnail = "false";
			}

			// Create a solr representation

			// Use all of the values that we've got from the file and modified as appropriate
			java.util.List<SOLRHelper.NVPair> index_properties = new java.util.ArrayList<SOLRHelper.NVPair>();
			for(String key: xpathMappings.keySet()) {
				if ( !"isPartOf".equals(key) ) {
					NameXPathMapping thisEntry = xpathMappings.get(key);
					if ( !"NO_INDEX".equals(thisEntry.getSolrName()) ) {

						String[] values = thisEntry.getValues();
						if ( values != null ) {
							if ( thisEntry.collectAll ) {
								// Store several values
								for(String aValue: values) {
									if ( aValue != null ) {
										index_properties.add(new SOLRHelper.NVPair(thisEntry.getSolrName(), aValue));
									}
								}
							} else {
								// Store just one (if there is one)
								if ( values[0] != null) {
									index_properties.add(new SOLRHelper.NVPair(thisEntry.getSolrName(), values[0]));
								}
							}
						}
					}
				}
			}
			
			// Parse the spatial information if we have any
			//if ( xpathMappings.containsKey("spatial") && xpathMappings.get("spatial").getValues() != null ) {
				//ParseSpatialData spatialParser = new ParseSpatialData();
				//HashMap<String,String[]> spatialDataToIndex = spatialParser.parseSpatialData(xpathMappings.get("spatial").getValues(), "General");
				//
				//if ( spatialDataToIndex.size() > 0 ) {
					//// We have something to index - do it
					//for(String key: spatialDataToIndex.keySet()) {
						//String[] data = spatialDataToIndex.get(key);
						//for(String thisOne: data) {
							//index_properties.add(new SOLRHelper.NVPair(key, thisOne));
						//}
					//}
				//}
			//}


			// Now add the other values that don't come directly from the file
			if ( "true".equals(haveThumbnail) ) {
				index_properties.add(new SOLRHelper.NVPair("cached_thumbnail", possiblePathToCachedImage));
			}

			index_properties.add(new SOLRHelper.NVPair("have_thumbnail", haveThumbnail));

			String combinedSubject = "";
			String[] separateSubjects = {"subject.person", "subject.organisation", "subject.event"};
			for(String subject: separateSubjects) {
				NameXPathMapping subjectEntry = xpathMappings.get(subject);
				if ( subjectEntry != null && subjectEntry.getValues() != null && subjectEntry.getValues()[0] != null ) {
					combinedSubject += " " + subjectEntry.getValues()[0];
				}
			}
			index_properties.add(new SOLRHelper.NVPair("subject.combined", combinedSubject.trim()));

			for(String thisCollection: collectionList ) {
				if (thisCollection != null && !"".equals(thisCollection) ) {
					index_properties.add(new SOLRHelper.NVPair("dcterms.isPartOf", thisCollection));
				}
			}

			index_properties.add(new SOLRHelper.NVPair("authority", owner_id));
			index_properties.add(new SOLRHelper.NVPair("depositor_id", depositor_id));
			index_properties.add(new SOLRHelper.NVPair("record_type", "collection"));
			index_properties.add(new SOLRHelper.NVPair("restp","ServiceProvider"));
			index_properties.add(new SOLRHelper.NVPair("aggregator.internal.id",resourceId+""));

			String internalRecordTypeLink = "/dpp/resource/" + resourceId + "/stream/oai_dc"; // TODO - what type should this be?
			index_properties.add(new SOLRHelper.NVPair("aggregator.internal_record_link", internalRecordTypeLink));

			// Index the document with solr and store it in the database
			com.k_int.aggregator.util.SOLRHelper solr_helper = (com.k_int.aggregator.util.SOLRHelper) ctx.getBean("SolrHelper");

			Document solr_doc = solr_helper.postIndexEntries(index_properties, false);

			if (solr_doc != null) {
				content_store.store(XMLUtil.serializeDocument(solr_doc), owner_id, depositor_id, doc_identifier,
						"solr", "application/xml", "", authoritative, collectionList.toArray(new String[collectionList.size()]), noChecksumOptions);
			} else {
				log.debug("An error occurred when saving the document to solr - no document returned!");
				// TODO - should i change the return value? 
			}
			
	        returnValue.setDocId(resourceId);

			
		} catch (TransformerException te) {
			log.error("Problem - transformer exception: " + te);
			returnValue.setMessage("TransformerException thrown when parsing the OAI_COMPLETE document");
		} catch (XPathExpressionException xe) {
			log.error("Problem when parsing the OAI_COMPLETE document. " + xe.getMessage());
			returnValue.setMessage(xe.getMessage());
		} 

        return returnValue;
	}

	public void onApplicationEvent(ApplicationEvent evt) {
		log.info("Spring event : "+evt);
	}

	public void setApplicationContext(ApplicationContext ctx)
			throws BeansException {
		this.ctx = ctx;
	}
	
	private HashMap<String,NameXPathMapping> setupMappings() {

		HashMap<String, NameXPathMapping> mappings = new HashMap<String,NameXPathMapping>();

		mappings.put("DCollection-description",new NameXPathMapping("dc.description","/oai_complete:michael-notice/michael:digital-collection/michael:description/michael:abstract/michael:p",false));
		mappings.put("Projects-description",new NameXPathMapping("dc.description","/oai_complete:michael-notice/michael:project/michael:description/michael:abstract/michael:p",false));
		mappings.put("Service-description",new NameXPathMapping("dc.description","/oai_complete:michael-notice/michael:service/michael:description/michael:abstract/michael:p",false));
		mappings.put("DCollection-identifier", new NameXPathMapping("dc.identifier","/oai_complete:michael-notice/michael:digital-collection/@id",false));
		mappings.put("Projects-identifier", new NameXPathMapping("dc.identifier","/oai_complete:michael-notice/michael:project/@id", false));
		mappings.put("Institution-identifier", new NameXPathMapping("dc.identifier","/oai_complete:michael-notice/michael:institution/@id", false));
		mappings.put("Service-identifier", new NameXPathMapping("dc.identifier","/oai_complete:michael-notice/michael:service/@id", false));
		mappings.put("DCollection-spatial",new NameXPathMapping("dcterms.spatial","/oai_complete:michael-notice/michael:digital-collection/michael:subject-indexing/michael:spatial-coverage-group/michael:spatial-coverage/michael:en",false, true)); 
		mappings.put("DCollection-title",new NameXPathMapping("dc.title","/oai_complete:michael-notice/michael:digital-collection/michael:identification/michael:title",false));
		mappings.put("Projects-title",new NameXPathMapping("dc.title","/oai_complete:michael-notice/michael:project/michael:identification/michael:title",false));
		mappings.put("Institution-title",new NameXPathMapping("dc.title","/oai_complete:michael-notice/michael:institution/michael:identification/michael:name",false));
		mappings.put("Service-title",new NameXPathMapping("dc.title","/oai_complete:michael-notice/michael:service/michael:identification/michael:title",false));
		
		return mappings;
	}


	/**
	 * Check through the data in the mappings (once filled with values) - if something is
	 * required then check that it isn't null i.e. that we have found a value for it
	 * @throws XPathExpressionException thrown if a field that is required has not been filled in from the XML document
	 */
	private void checkForRequiredData(HashMap<String,NameXPathMapping> xpathMappings) throws XPathExpressionException {
		// Loop through all of the entries and consider those where required == true
		for(String key: xpathMappings.keySet()) {
			NameXPathMapping thisEntry = xpathMappings.get(key);
			if ( thisEntry.isRequired() ) {
				// We care about it being a non-null value - check
				String[] value = thisEntry.getValues();
				if ( value == null ) {
					log.debug("The required field: " + key + " doesn't have a value in the data file");
					throw(new XPathExpressionException("Required field: " + key + " does not have a value in the data"));
				}
			}
		}
	}
	  
	/**
	 * Loop through the retrieved data and normalise it if appropriate
	 * 
	 * @param xpathMappings The list of mappings and data to normalise
	 * @return The normalised list of mappings and data
	 */
	private HashMap<String,NameXPathMapping> normaliseData(HashMap<String,NameXPathMapping> xpathMappings) {

		log.debug("In normaliseData");
		Set<String> keySet = xpathMappings.keySet();

		log.debug("keySet size: " + keySet.size());

		// Normalise the dcmi:type of the data
		if ( keySet.contains("type") ) {
			// There is a type entry, does it have a value / values?
			NameXPathMapping typeMapping = xpathMappings.get("type");
			if ( typeMapping != null) {
				String[] types = typeMapping.getValues();
				if ( types == null ) {
					types = new String[1];
					types[0] = "Unknown";
				}
				List<String> newTypes = new ArrayList<String>();

				for(int ctr = 0; ctr < types.length; ctr++) {
					String[] tempTypes = NormaliseTypes.normalise(types[ctr]);
					for(int ctr2 = 0; ctr2 < tempTypes.length; ctr2++) {
						newTypes.add(tempTypes[ctr2]);
					}
				}

				String[] newTypesAsArray = newTypes.toArray(new String[0]);

				// Remember the normalised values
				typeMapping.setValues(newTypesAsArray);
				xpathMappings.put("type", typeMapping);
			}
		}
		
		// Normalise the title - remove ^ from the string
		if ( keySet.contains("title") ) {
			// There is a title entry
			NameXPathMapping titleMapping = xpathMappings.get("title");
			if ( titleMapping != null ) {
				String[] titles = titleMapping.getValues();
				if ( titles == null || titles.length == 0 || titles[0] == null) {
					titles = new String[1];
					titles[0] = "Unknown";
				}
				
				if ( titles[0].contains("^") ) {
					titles[0] = titles[0].replaceAll("\\^", "");
				}

				titleMapping.setValues(titles);
				xpathMappings.put("title", titleMapping);
			}
		}

		return xpathMappings;
	}


	/**
	 * Get the value of the element or attribute given by the specified xpath expression
	 * @param metadata_record The document to apply the xpath to
	 * @param xpath The xpath expression to the element or attribute to consider
	 * @param namespace_node The set of namespaces used in the xpath and document
	 * @return The text contents of the elements or attributes as an array
	 * @throws javax.xml.transform.TransformerException thrown if there's an error with the XPath
	 */
	protected static String[] getValues(Node metadata_record, String xpath, Node namespace_node) throws javax.xml.transform.TransformerException {
		// First work out whether we're looking for an attribute or a full element
		// - we're looking at an attribute if there is an @ after the last / 
		// (and the expression doesn't end with ].
		String node_xpath = xpath;
		String attribute_xpath = "";

		if ( !xpath.endsWith("]") ) {

			int indexOfLastSlash = xpath.lastIndexOf("/");
			int indexOfLastAt = xpath.lastIndexOf("@");

			if ( indexOfLastAt > indexOfLastSlash ) {
				// Dealing with an attribute
				attribute_xpath = xpath.substring(indexOfLastAt + 1);
				node_xpath = xpath.substring(0, indexOfLastAt - 1);
			}
		}

		// Go and get the node(s) we're interested in
		NodeIterator nodeList = XPathAPI.selectNodeIterator(metadata_record, node_xpath, namespace_node);
		String[] returnValue = null;
		ArrayList<String> tempList = new ArrayList<String>();

		Node actualNode;
		while ((actualNode = nodeList.nextNode()) != null) {
			// We've got a node - now are we getting an attribute or not?
			String value = null;
			if ( !"".equals(attribute_xpath) ) {
				value = ((Element)actualNode).getAttribute(attribute_xpath);
			} else {
				value = extractText(actualNode);
			}

			// Unescape any html entites, etc. in the string (a couple of times since there are some places
			// that encode multiple times and then remove the HTML
			//			  log.debug("about to unescape the value: " + value);
			if ( value != null ) {
				value = StringEscapeUtils.unescapeHtml(StringEscapeUtils.unescapeHtml(value));
				value = value.replaceAll("\\<.*?\\>", "");
			}

			//			  log.debug("value after unescaping and stripping tags: " + value);

			tempList.add(value);
		}

		returnValue = (String[])tempList.toArray(new String[tempList.size()]);

		return returnValue;
	}


	protected static String extractText(Node n) {
		try {
			Node node = XPathAPI.selectSingleNode(n,"./text()");
			if ( node != null ) {
				node.normalize();
				return node.getNodeValue().trim();
			}
		}
		catch ( javax.xml.transform.TransformerException te ) {
			te.printStackTrace();
		}
		return null;
	}
	
	private HashMap<String,NameXPathMapping> normaliseMichaelDataFormats(HashMap<String,NameXPathMapping> mappings) {

		HashMap<String, NameXPathMapping> newMappings = new HashMap<String, NameXPathMapping>();
		
		// Merge the different identifiers into a single 'identifier' entry
		if (mappings.containsKey("DCollection-identifier")	&& mappings.get("DCollection-identifier").getValues() != null) {
			newMappings.put("identifier", mappings.get("DCollection-identifier"));
		} else if (mappings.containsKey("Projects-identifier")	&& mappings.get("Projects-identifier").getValues() != null) {
			 newMappings.put("identifier", mappings.get("Projects-identifier"));
		} else if ( mappings.containsKey("Institution-identifier")	&& mappings.get("Institution-identifier").getValues() != null) {
			 newMappings.put("identifier", mappings.get("Institution-identifier"));
		} else if ( mappings.containsKey("Service-identifier")	&& mappings.get("Service-identifier").getValues() != null) {
			 newMappings.put("identifier", mappings.get("Service-identifier"));
		}
		
		// Merge the different descriptions into a single 'description' entry
		if (mappings.containsKey("DCollection-description")	&& mappings.get("DCollection-description").getValues() != null) {
			newMappings.put("description", mappings.get("DCollection-description"));
		} else if (mappings.containsKey("Projects-description")	&& mappings.get("Projects-description").getValues() != null) {
			newMappings.put("description", mappings.get("Projects-description"));
		} else if ( mappings.containsKey("Service-description")	&& mappings.get("Service-description").getValues() != null) {
			newMappings.put("description", mappings.get("Service-description"));
		}
		
		// Merge the different spatial information into a single 'spatial' entry
		if ( mappings.containsKey("DCollection-spatial") && mappings.get("DCollection-spatial").getValues() != null ) {
			newMappings.put("spatial", mappings.get("DCollection-spatial"));
		}
		
		// Merge the different title information into a single 'title' entry
		if (mappings.containsKey("DCollection-title")	&& mappings.get("DCollection-title").getValues() != null) {
			newMappings.put("title", mappings.get("DCollection-title"));
		} else if (mappings.containsKey("Projects-title")	&& mappings.get("Projects-title").getValues() != null) {
			newMappings.put("title", mappings.get("Projects-title"));
		} else if ( mappings.containsKey("Institution-title")	&& mappings.get("Institution-title").getValues() != null) {
			newMappings.put("title", mappings.get("Institution-title"));
		} else if ( mappings.containsKey("Service-title")	&& mappings.get("Service-title").getValues() != null) {
			newMappings.put("title", mappings.get("Service-title"));
		}
		
		return newMappings;
		
	}

}