package com.k_int.discover.service; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Set; import javax.xml.transform.TransformerException; import javax.xml.xpath.XPathExpressionException; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.xpath.XPathAPI; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ApplicationEvent; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.traversal.NodeIterator; import com.k_int.aggregator.cache.CacheContentResult; import com.k_int.aggregator.cache.impl.CacheContentImageThumbnail; import com.k_int.aggregator.core.DepositResult; import com.k_int.aggregator.plugin.XMLDocumentHandlerPlugin; import com.k_int.aggregator.plugin.XMLProcessingResult; import com.k_int.aggregator.repository.RepositoryService; import com.k_int.aggregator.repository.RepositoryStoreOptions; import com.k_int.aggregator.repository.RepositoryStoreResult; import com.k_int.aggregator.util.SOLRHelper; import com.k_int.commons.util.XMLUtil; import com.k_int.discover.datamodel.OAIDCDocument; import com.k_int.discover.datamodel.CultureGrid_ItemDocument; import com.k_int.discover.util.ParseSpatialData; @Service("XMLHandler-PNDSDCAP") public class PNDSDCAPHandler implements XMLDocumentHandlerPlugin, ApplicationContextAware, org.springframework.context.ApplicationListener { private static Log log = LogFactory.getLog(PNDSDCAPHandler.class); protected ApplicationContext ctx = null; // Namespace declarations public static String XMLNS_DC = "http://purl.org/dc/elements/1.1/"; public static String XMLNS_DCTERMS = "http://purl.org/dc/terms/"; public static String XMLNS_PNDSTERMS = "http://purl.org/mla/pnds/terms/"; public static String XMLNS_XSI = "http://www.w3.org/2001/XMLSchema-instance"; public static String XMLNS_PNDS_DC = "http://purl.org/mla/pnds/pndsdc/"; public static String XMLNS_OAI_DC = "http://www.openarchives.org/OAI/2.0/oai_dc/"; public static String XMLNS_XHTML = "http://www.w3.org/1999/xhtml"; public static String XMLNS_PNDS_DC_E20CL = "http://www.20thcenturylondon.org.uk"; private static String[] SUPPORTED_SCHEMAS = new String[] { XMLNS_PNDS_DC, XMLNS_PNDS_DC_E20CL }; public String[] getSupportedSchemas() { return SUPPORTED_SCHEMAS; } /** * @deprecated Replaced by {@link #process(DepositResult, byte[], Document, String, String, String, Boolean, long)} */ @Deprecated public XMLProcessingResult process(DepositResult deposit_result, byte[] content, Document d, String depositor_id, String owner_id, String root_namespace, Boolean authoritative) { return this.process(deposit_result, content, d, depositor_id, owner_id, root_namespace, authoritative, (RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL | RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM | RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) ); } public XMLProcessingResult process(DepositResult deposit_result, byte[] content, Document d, String depositor_id, String owner_id, String root_namespace, Boolean authoritative, long options) { // Set up the return value XMLProcessingResult returnValue = new XMLProcessingResult(); // Set up a version of the options with store checksums set to false as used when // storing all versions of the document other than the original long noChecksumOptions = options & ~RepositoryStoreOptions.REPO_STORE_OPTIONS_CALC_CHECKSUM; // Set up namespace elements Element new_namespace_node = d.createElement("NSNode"); // Set up our local namespace definitions new_namespace_node.setAttribute("xmlns:dc", XMLNS_DC); new_namespace_node.setAttribute("xmlns:dcterms", XMLNS_DCTERMS); new_namespace_node.setAttribute("xmlns:xsi", XMLNS_XSI); new_namespace_node.setAttribute("xmlns:pn", XMLNS_PNDS_DC); new_namespace_node.setAttribute("xmlns:pndsterms", XMLNS_PNDSTERMS); new_namespace_node.setAttribute("xmlns:oai_dc", XMLNS_OAI_DC); new_namespace_node.setAttribute("xmlns:pn", XMLNS_PNDS_DC); new_namespace_node.setAttribute("xmlns:e20cl", XMLNS_PNDS_DC_E20CL); try { // Get all of the data from the original file HashMap xpathMappings = this.setupMappings(); for(String key: xpathMappings.keySet()) { NameXPathMapping thisOne = xpathMappings.get(key); String xpath = thisOne.getXpathExpression(); String[] newValues = null; newValues = getValues(d, xpath, new_namespace_node); // log.debug("Just mapped xpath " + xpath + " to " + newValues.length + " values"); if (newValues != null && newValues.length != 0 ) { thisOne.setValues(newValues); // log.debug("stored the new found value.. in a local object"); xpathMappings.put(key, thisOne); // log.debug("stored the value in the mappings"); } } String doc_identifier = xpathMappings.get("identifier").getValues()[0]; log.debug("doc_identifier: " + doc_identifier); // Perform some modifications for the data as required by the different providers to // make their data correct NameXPathMapping relatedLinkEntry = new NameXPathMapping("dc.related.link", "", false); if ( doc_identifier.startsWith("http://") || doc_identifier.startsWith("https://") ) { String[] relLink = {doc_identifier}; relatedLinkEntry.setValues(relLink); } // Change 1 - e20cl thumbnails // Do some mangling for e20cl where we need to change the identifier and thumbnail urls // and also for collect britain where the URLs have changed NameXPathMapping thumbnailEntry = xpathMappings.get("thumbnail"); NameXPathMapping identifierEntry = xpathMappings.get("identifier"); NameXPathMapping type = xpathMappings.get("type"); // If we're looking at e20cl then we can set the thumbnail based on the identifier if ( identifierEntry != null && identifierEntry.getValues() != null && identifierEntry.getValues()[0].startsWith("http://www.20thcenturylondon.org.uk/rserver.php?") && type != null && type.getValues() != null && "image".equalsIgnoreCase(type.getValues()[0]) && (thumbnailEntry == null || thumbnailEntry.getValues() == null || thumbnailEntry.getValues()[0] == null) ) { // We are looking at e20cl and have an image that we want to get the thumbnail for, // but we don't have a thumbnail at the moment String[] thumbnailURL = new String[1]; String tempString = identifierEntry.getValues()[0]; tempString = tempString.replace("rserver.php", "thumbfor.php"); thumbnailURL[0] = tempString; // log.debug("about to set the thumbnail value to : " + tempString); thumbnailEntry.setValues(thumbnailURL); xpathMappings.put("thumbnail", thumbnailEntry); } //NOTE - the above won't work if 20th Century London ever go and change their URL... // Change 2 - Collect britain identifiers and thumbnails // If we're looking at collect britain then we need to change the thumbnail links // as they're wrong in the data we have on file.. and we need to map the identifier to a // different url for use as the link if ( identifierEntry != null && identifierEntry.getValues() != null && identifierEntry.getValues()[0].startsWith("http://www.collectbritain.co.uk/personalisation") ) { // We're looking at collect britain.. // NOTE - the following won't work if collect britain URLs change again... // Map the identifier from 'http://www.collectbritain.co.uk/personalisation/object.cfm?uid=********' // to 'http://www.bl.uk/onlinegallery/onlineex/unvbrit/ZZ/######.html' where ###### == ****.toLowerCase() // and ZZ == the first character of the object's title.toLowerCase(); in order to work out the related // link String titleFirstChar = "other"; char firstChar = xpathMappings.get("title").getValues()[0].charAt(0); if ( Character.isLetter(firstChar) ) { titleFirstChar = Character.toString(firstChar); } String replacementExhib = ""; String isPartOf = xpathMappings.get("isPartOf").getValues()[0]; if ( isPartOf.contains("/19thphotobooks/") ) { replacementExhib = "earlyphotos"; } else if ( isPartOf.contains("/caribbean/") ) { replacementExhib = "carviews"; } else if ( isPartOf.contains("/crace/") ) { replacementExhib = "crace"; } else if (isPartOf.contains("/deptford/") ) { replacementExhib = "deptford"; } else if (isPartOf.contains("/evanion/") ) { replacementExhib = "evancoll"; } else if (isPartOf.contains("/kenturnpike/") ) { replacementExhib = "kensturn"; } else if (isPartOf.contains("/ktop/") ) { replacementExhib = "kinggeorge"; } else if (isPartOf.contains("/osd/") ) { replacementExhib = "ordsurvdraw"; } else if (isPartOf.contains("/philatelic/") ) { replacementExhib = "philvar"; } else if (isPartOf.contains("/topdrawings/") ) { replacementExhib = "topdrawings"; } else if (isPartOf.contains("/unveiling/") ) { replacementExhib = "unvbrit"; } else if (isPartOf.contains("vicmusic") ) { replacementExhib = "vicpopmus"; } else { // Something we don't know how to map replacementExhib = "UNMAPPABLE"; log.debug("We don't know how to map the address from the dataset to the new BL address. isPartOf: " + isPartOf); } String replacementURL = "http://www.bl.uk/onlinegallery/onlineex/" + replacementExhib + "/" + titleFirstChar + "/"; String[] relatedLink = {identifierEntry.getValues()[0].replace("http://www.collectbritain.co.uk/personalisation/object.cfm?uid=", replacementURL) + ".html"}; relatedLink[0] = relatedLink[0].toLowerCase(); // If we couldn't map the collection name then return it to the original value if ( replacementExhib.equals("UNMAPPABLE") ) { relatedLink[0] = relatedLinkEntry.getValues()[0]; } // log.debug("Modified collect britain identifier is now: " + newIdentifier[0]); relatedLinkEntry.setValues(relatedLink); // Now check if we should change the thumbnail.. if ( type != null && type.getValues() != null && "image".equalsIgnoreCase(type.getValues()[0]) ) { // We're looking at an image so we need to map the thumbnail url from // 'http://www.collectbritain.com/mediastore/***/###/XYZ.JPG' to // 'http://ogimages.bl.uk/images/***/XYZ.jpg' String[] newThumbnail = {thumbnailEntry.getValues()[0].replace("http://www.collectbritain.com/mediastore", "http://ogimages.bl.uk/images")}; newThumbnail[0].replace(".JPG", ".jpg"); // Remove the /###/ part of the URL (replacing it with just / int indexOfLastSlash = newThumbnail[0].lastIndexOf("/"); int indexOfLastButOneSlash = newThumbnail[0].substring(0,indexOfLastSlash).lastIndexOf("/"); if ( indexOfLastSlash == indexOfLastButOneSlash ) { log.debug("Error finding the last two slashes to remove everything between them - found the same one twice!"); } else if ( indexOfLastSlash < 0 || indexOfLastButOneSlash < 0 ) { log.debug("Unable to find the last two slashes at all - indices: " + indexOfLastSlash + " " + indexOfLastButOneSlash ); } else { newThumbnail[0] = newThumbnail[0].substring(0,indexOfLastButOneSlash) + newThumbnail[0].substring(indexOfLastSlash); } // log.debug("Modified collect britain thumbnail is now: " + newThumbnail[0]); thumbnailEntry.setValues(newThumbnail); xpathMappings.put("thumbnail", thumbnailEntry); } } // Change 3 - Farne identifiers // Now check to see if we're looking at farne files and map the identifier to point // to the actual site for the related link if we are if ( identifierEntry != null && identifierEntry.getValues() != null && identifierEntry.getValues()[0].startsWith("farne:") ) { // We're looking at farne.. // NOTE - the following won't work if farne URLs change... // Map from 'farne:####' to 'http://www.asaplive.com/archive/detail.asp?id=####' String replacementURL = "http://www.asaplive.com/archive/detail.asp?id="; String[] relatedLink = { identifierEntry.getValues()[0].replace("farne:", replacementURL) }; // log.debug("Modified farne identifier is now: " + newIdentifier[0]); relatedLinkEntry.setValues(relatedLink); } // Change 4 - Picture the past or Kirklees images thumbnails // Now check to see if we're looking at Picture the past or Kirklees Images and change the thumbnail link // to point at the correct site if ( owner_id.equals("KirkleesImages") || owner_id.equals("PictureThePast") ) { // We're looking at picture the past or Kirklees images // NOTE - the following won't work if picture the past URLs change... String originalURL = "https://system.hpacde.org.uk/"; String replacementURL = "https://www.hpacde.org.uk/"; if ( thumbnailEntry != null && thumbnailEntry.getValues() != null && thumbnailEntry.getValues()[0] != null ) { // We have a thumbnail to replace String[] newThumbnail = { thumbnailEntry.getValues()[0].replace(originalURL, replacementURL) }; // log.debug("Modified Picture the Past thumbnail is now: " + newThumbnail[0]); thumbnailEntry.setValues(newThumbnail); xpathMappings.put("thumbnail", thumbnailEntry); } } // We've got a final value for a related link now (which may be null), but remember it xpathMappings.put("relatedLink", relatedLinkEntry); // Modify and / or normalise the data as appropriate xpathMappings = this.normaliseData(xpathMappings); // Check that we have all of the required information this.checkForRequiredData(xpathMappings); // Work out the list of collections for the data List collectionList = new ArrayList(); if ( "BowesOAI".equals(owner_id) || "DevonEtched".equals(owner_id) || "DCDA".equals(owner_id) || "Leodis".equals(owner_id) || "Farne".equals(owner_id) || "TWImagine".equals(owner_id) || "Commanet".equals(owner_id) || "SOPSE".equals(owner_id) || "KirkleesImages".equals(owner_id) || "PictureThePast".equals(owner_id) || "DurhamCountyCouncil".equals(owner_id) ||"BeamishTreasures".equals(owner_id) || "BedesWorld".equals(owner_id) || "iSeeGateshead".equals(owner_id) || "SevenStories".equals(owner_id) || "FitzwilliamMuseum".equals(owner_id) ) { collectionList.add(owner_id); } else if ( "CollectBritain".equals(owner_id) ) { collectionList.add("CollectBritain"); // TODO - add more specific collections that relate to the initial isPartOf } else if ( "Bromley".equals(owner_id) || "Bishopsgate".equals(owner_id) || "Brent".equals(owner_id) || "Croydon".equals(owner_id) || "Hampstead".equals(owner_id) || "Haringey".equals(owner_id) || "MOL".equals(owner_id) || "LTM".equals(owner_id) || "GMUS".equals(owner_id) || "HornimanMuseum".equals(owner_id) || "JewishMuseum".equals(owner_id) || "RugbyMuseum".equals(owner_id) || "RedbridgeLocalStudies".equals(owner_id) || "RedbridgeMuseum".equals(owner_id) ) { collectionList.add(owner_id); collectionList.add("e20cl"); } else if ( "MDA".equals(owner_id) ) { collectionList.add("MoDA"); collectionList.add("e20cl"); } else { log.debug("Unable to determine the collection list based on the source of the data. Source: " + owner_id); } // Set up the objects needed to store the different documents RepositoryService content_store = (RepositoryService) ctx.getBean("ContentStore"); Long resourceId = -1l; // Store the Original format (if specified in options) if ( (options & RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC) == RepositoryStoreOptions.REPO_STORE_OPTIONS_STORE_ORIGINAL_DOC ) { RepositoryStoreResult store_result = content_store.store(content,owner_id,depositor_id,doc_identifier,"pnds_dcap_raw","application/xml", SUPPORTED_SCHEMAS[0],authoritative, collectionList.toArray(new String[collectionList.size()]), options); resourceId = store_result.getDocId(); } else { log.debug("Not storing the original document, as specified by the options"); } // Create an OAI_DC representation OAIDCDocument oaiDcDoc = new OAIDCDocument(); if ( xpathMappings.containsKey("description") && xpathMappings.get("description").getValues() != null && xpathMappings.get("description").getValues().length > 0 ) { oaiDcDoc.setDescription(xpathMappings.get("description").getValues()[0]); } if ( xpathMappings.containsKey("identifier") && xpathMappings.get("identifier").getValues() != null && xpathMappings.get("identifier").getValues().length > 0 ) { oaiDcDoc.setIdentifier(xpathMappings.get("identifier").getValues()[0]); } if ( xpathMappings.containsKey("subject") && xpathMappings.get("subject").getValues() != null && xpathMappings.get("subject").getValues().length > 0 ) { List subjects = new ArrayList(); String[] tempSubjectArray = xpathMappings.get("subject").getValues(); for(String thisSubject: tempSubjectArray ) { subjects.add(thisSubject); } oaiDcDoc.setSubject(subjects); } if ( xpathMappings.containsKey("title") && xpathMappings.get("title").getValues() != null && xpathMappings.get("title").getValues().length > 0 ) { oaiDcDoc.setTitle(xpathMappings.get("title").getValues()[0]); } if ( xpathMappings.containsKey("type") && xpathMappings.get("type").getValues() != null && xpathMappings.get("type").getValues().length > 0 ) { oaiDcDoc.setType(xpathMappings.get("type").getValues()[0]); } oaiDcDoc.setCreator(owner_id); // Store the OAI_DC version log.debug("Storing the generated oai_dc document in the content store"); byte[] serialisedOaiDc = XMLUtil.serializeDocument(oaiDcDoc.toXML()); if ( serialisedOaiDc != null ) { RepositoryStoreResult store_result = content_store.store(serialisedOaiDc,owner_id, depositor_id, doc_identifier, "oai_dc", "application/xml", "http://www.openarchives.org/OAI/2.0/oai_dc/", authoritative, collectionList.toArray(new String[collectionList.size()]), noChecksumOptions); if ( resourceId == -1l ) { // We didn't have a resource ID before, remember it now resourceId = store_result.getDocId(); } } // Create a CultureGrid_Item representation CultureGrid_ItemDocument cgItemDoc = new CultureGrid_ItemDocument(); cgItemDoc.setOwner(owner_id); if ( xpathMappings.containsKey("identifier") && xpathMappings.get("identifier").getValues() != null && xpathMappings.get("identifier").getValues().length > 0 ) { cgItemDoc.setIdentifier(xpathMappings.get("identifier").getValues()[0]); } if ( xpathMappings.containsKey("title") && xpathMappings.get("title").getValues() != null && xpathMappings.get("title").getValues().length > 0 ) { cgItemDoc.setTitle(xpathMappings.get("title").getValues()[0]); } if ( xpathMappings.containsKey("description") && xpathMappings.get("description").getValues() != null && xpathMappings.get("description").getValues().length > 0 ) { cgItemDoc.setDescription(xpathMappings.get("description").getValues()[0]); } if ( xpathMappings.containsKey("license") && xpathMappings.get("license").getValues() != null && xpathMappings.get("license").getValues().length > 0 ) { cgItemDoc.setLicense(xpathMappings.get("license").getValues()[0]); } if ( xpathMappings.containsKey("licenseText") && xpathMappings.get("licenseText").getValues() != null && xpathMappings.get("licenseText").getValues().length > 0 ) { cgItemDoc.setLicenseText(xpathMappings.get("licenseText").getValues()[0]); } if ( xpathMappings.containsKey("rightsHolder") && xpathMappings.get("rightsHolder").getValues() != null && xpathMappings.get("rightsHolder").getValues().length > 0 ) { cgItemDoc.setRightsHolder(xpathMappings.get("rightsHolder").getValues()[0]); } if ( xpathMappings.containsKey("subject") && xpathMappings.get("subject").getValues() != null && xpathMappings.get("subject").getValues().length > 0 ) { List subjects = new ArrayList(); String[] tempSubjectArray = xpathMappings.get("subject").getValues(); for(String thisSubject: tempSubjectArray ) { subjects.add(thisSubject); } cgItemDoc.setSubject(subjects); } if ( xpathMappings.containsKey("type") && xpathMappings.get("type").getValues() != null && xpathMappings.get("type").getValues().length > 0 ) { List types = new ArrayList(); String[] tempTypesArray = xpathMappings.get("type").getValues(); for(String thisType: tempTypesArray ) { types.add(thisType); } cgItemDoc.setType(types); } if ( xpathMappings.containsKey("spatial") && xpathMappings.get("spatial").getValues() != null && xpathMappings.get("spatial").getValues().length > 0 ) { cgItemDoc.setSpatial(xpathMappings.get("spatial").getValues()[0]); } if ( xpathMappings.containsKey("temporal") && xpathMappings.get("temporal").getValues() != null && xpathMappings.get("temporal").getValues().length > 0 ) { cgItemDoc.setTemporal(xpathMappings.get("temporal").getValues()[0]); } if ( xpathMappings.containsKey("thumbnail") && xpathMappings.get("thumbnail").getValues() != null && xpathMappings.get("thumbnail").getValues().length > 0 ) { cgItemDoc.setThumbnail(xpathMappings.get("thumbnail").getValues()[0]); } else if ( xpathMappings.containsKey("thumbnail2") && xpathMappings.get("thumbnail2").getValues() != null && xpathMappings.get("thumbnail2").getValues().length > 0 ) { cgItemDoc.setThumbnail(xpathMappings.get("thumbnail2").getValues()[0]); } if ( xpathMappings.containsKey("relatedLink") && xpathMappings.get("relatedLink").getValues() != null && xpathMappings.get("relatedLink").getValues().length > 0 ) { cgItemDoc.setRelatedLink(xpathMappings.get("relatedLink").getValues()[0]); } if ( collectionList != null ) { cgItemDoc.setIsPartOf(collectionList); } // Store the PNDS_DCAP version log.debug("Storing the generated pnds_dcap document in the content store"); byte[] serialisedPndsDcapDoc = XMLUtil.serializeDocument(cgItemDoc.toXML()); if ( serialisedPndsDcapDoc != null ) { content_store.store(serialisedPndsDcapDoc,owner_id, depositor_id, doc_identifier, "CultureGrid_Item", "application/xml", "http://www.peoplesnetwork.gov.uk/schema/CultureGrid_Item", authoritative, collectionList.toArray(new String[collectionList.size()]), noChecksumOptions); // TODO - schema / document type.. } // Download and create a thumbnail, storing it in the db if appropriate (and if specified in options) NameXPathMapping thumbnail2Entry = xpathMappings.get("thumbnail2"); String haveThumbnail = "true"; String possiblePathToCachedImage = "/dpp/resource/" + resourceId + "/stream/thumbnail_image_jpeg"; if ( (thumbnailEntry != null && thumbnailEntry.getValues() != null && thumbnailEntry.getValues()[0] != null) || (thumbnail2Entry != null && thumbnail2Entry.getValues() != null && thumbnail2Entry.getValues()[0] != null) ) { // Go and cache the thumbnail if specified to do so in the options if ( (options & RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL) == RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL ) { // Work out which of the values to use for the path to the thumbnail String fullImageAddress = ""; if ( thumbnailEntry.getValues() != null && thumbnailEntry.getValues()[0] != null && !"".equals(thumbnailEntry.getValues()[0]) ) { fullImageAddress = thumbnailEntry.getValues()[0]; } else { fullImageAddress = thumbnail2Entry.getValues()[0]; } log.debug("Address of image to be cached: " + fullImageAddress); // Go and download the thumbnail resizing it as appropriate and then store it into the database CacheContentResult cachedThumbnail = null; CacheContentImageThumbnail resizer = new CacheContentImageThumbnail(); cachedThumbnail = resizer.cacheObject(fullImageAddress); if ( cachedThumbnail != null ) { // Store the thumbnail in the database and the path to it in the solr content_store.store(cachedThumbnail.content,owner_id, depositor_id, doc_identifier, cachedThumbnail.type, cachedThumbnail.mimeType, cachedThumbnail.subType, authoritative,collectionList.toArray(new String[collectionList.size()]), noChecksumOptions); } else { haveThumbnail = "false"; } } else { // Look in the database to see if there's already a thumbnail for this record // and if there is use it instead Long thumbnailID = null; thumbnailID = content_store.lookupResourceInstanceId(resourceId, "thumbnail_image_jpeg"); if ( thumbnailID == null ) { haveThumbnail = "false"; } else { log.debug("Using previously cached thumbnail"); } } } else { haveThumbnail = "false"; } // Create a solr representation // Use all of the values that we've got from the file and modified as appropriate java.util.List index_properties = new java.util.ArrayList(); for(String key: xpathMappings.keySet()) { if ( !"isPartOf".equals(key) ) { NameXPathMapping thisEntry = xpathMappings.get(key); if ( !"NO_INDEX".equals(thisEntry.getSolrName()) ) { String[] values = thisEntry.getValues(); if ( values != null ) { if ( thisEntry.collectAll ) { // Store several values for(String aValue: values) { if ( aValue != null ) { index_properties.add(new SOLRHelper.NVPair(thisEntry.getSolrName(), aValue)); } } } else { // Store just one (if there is one) if ( values[0] != null) { index_properties.add(new SOLRHelper.NVPair(thisEntry.getSolrName(), values[0])); } } } } } } // Parse the spatial information if we have any // if ( xpathMappings.containsKey("spatial") && xpathMappings.get("spatial").getValues() != null ) { // if ( xpathMappings.containsKey("spatial-format") && xpathMappings.get("spatial-format").getValues() != null && xpathMappings.get("spatial-format").getValues().length > 0) { // String[] formats = xpathMappings.get("spatial-format").getValues(); // String[] values = xpathMappings.get("spatial").getValues(); // // if ( formats.length == values.length ) { // for(int ctr = 0; ctr < formats.length; ctr++) { // String thisFormat = formats[ctr]; // String[] thisValue = {values[ctr]}; // // HashMap spatialDataToIndex = new HashMap(); // ParseSpatialData spatialParser = new ParseSpatialData(); // // if ( thisFormat.equals("http://purl.org/dc/terms/TGN") ) { // spatialDataToIndex = spatialParser.parseSpatialData(thisValue, "Getty"); // } else if (thisFormat.equals("http://purl.org/mla/pnds/terms/OSGridRef") ) { // spatialDataToIndex = spatialParser.parseSpatialData(thisValue, "OSGridRef"); // } else { // log.info("Unable to work out what type of parsing to run on spatial data from a PNDS_DCAP document. Format: " + thisFormat); // } // // // if ( spatialDataToIndex.size() > 0 ) { // // We have something to index - do it // for(String key: spatialDataToIndex.keySet()) { // String[] data = spatialDataToIndex.get(key); // for(String thisOne: data) { // index_properties.add(new SOLRHelper.NVPair(key, thisOne)); // } // } // } // } // } else { // log.info("Unable to parse the spatial information from the file as we have different numbers of pieces of spatial information from spatial formats"); // } // // } // // } // if ( xpathMappings.containsKey("place") && xpathMappings.get("place").getValues() != null ) { // ParseSpatialData spatialParser = new ParseSpatialData(); // HashMap spatialDataToIndex = spatialParser.parseSpatialData(xpathMappings.get("place").getValues(), "General"); // // if ( spatialDataToIndex.size() > 0 ) { // // We have something to index - do it // for(String key: spatialDataToIndex.keySet()) { // String[] data = spatialDataToIndex.get(key); // for(String thisOne: data) { // index_properties.add(new SOLRHelper.NVPair(key, thisOne)); // } // } // } // // } // Now add the other values that don't come directly from the file if ( "true".equals(haveThumbnail) ) { index_properties.add(new SOLRHelper.NVPair("cached_thumbnail", possiblePathToCachedImage)); } index_properties.add(new SOLRHelper.NVPair("have_thumbnail", haveThumbnail)); String combinedSubject = ""; String[] separateSubjects = {"subject.person", "subject.organisation", "subject.event"}; for(String subject: separateSubjects) { NameXPathMapping subjectEntry = xpathMappings.get(subject); if ( subjectEntry != null && subjectEntry.getValues() != null && subjectEntry.getValues()[0] != null ) { combinedSubject += " " + subjectEntry.getValues()[0]; } } index_properties.add(new SOLRHelper.NVPair("subject.combined", combinedSubject.trim())); for(String thisCollection: collectionList ) { if (thisCollection != null && !"".equals(thisCollection) ) { index_properties.add(new SOLRHelper.NVPair("dcterms.isPartOf", thisCollection)); } } index_properties.add(new SOLRHelper.NVPair("authority", owner_id)); index_properties.add(new SOLRHelper.NVPair("depositor_id", depositor_id)); index_properties.add(new SOLRHelper.NVPair("record_type", "item")); index_properties.add(new SOLRHelper.NVPair("restp","ServiceProvider")); index_properties.add(new SOLRHelper.NVPair("aggregator.internal.id",resourceId+"")); String internalRecordTypeLink = "/dpp/resource/" + resourceId + "/stream/CultureGrid_Item"; index_properties.add(new SOLRHelper.NVPair("aggregator.internal_record_link", internalRecordTypeLink)); // Index the document with solr and store it in the database com.k_int.aggregator.util.SOLRHelper solr_helper = (com.k_int.aggregator.util.SOLRHelper) ctx.getBean("SolrHelper"); Document solr_doc = solr_helper.postIndexEntries(index_properties, false); if (solr_doc != null) { content_store.store(XMLUtil.serializeDocument(solr_doc), owner_id, depositor_id, doc_identifier, "solr", "application/xml", "", authoritative, collectionList.toArray(new String[collectionList.size()]), noChecksumOptions); } else { log.debug("An error occurred when saving the document to solr - no document returned!"); // TODO - should i change the return value? } returnValue.setDocId(resourceId); } catch (TransformerException te) { log.error("Problem - transformer exception: " + te); returnValue.setMessage("TransformerException thrown when parsing the OAI_DC document"); } catch (XPathExpressionException xe) { log.error("Problem when parsing the OAI_DC document. " + xe.getMessage()); returnValue.setMessage(xe.getMessage()); } return returnValue; } public void onApplicationEvent(ApplicationEvent evt) { log.info("Spring event : "+evt); } public void setApplicationContext(ApplicationContext ctx) throws BeansException { this.ctx = ctx; } private HashMap setupMappings() { HashMap mappings = new HashMap(); // mappings.put("audience", new NameXPathMapping("dcterms.audience","/pn:description/dcterms:audience", false)); // mappings.put("contributor", new NameXPathMapping("dc.contributor","/pn:description/dc:contributor", false)); mappings.put("description", new NameXPathMapping("dc.description","/pn:description/dc:description", false)); // mappings.put("format", new NameXPathMapping("dc.format","/pn:description/dc:format", false)); mappings.put("identifier", new NameXPathMapping("dc.identifier","/pn:description/dc:identifier", true)); mappings.put("isPartOf", new NameXPathMapping("dcterms.isPartOf", "/pn:description/dcterms:isPartOf/@valueURI", false)); // mappings.put("language", new NameXPathMapping("dc.language","/pn:description/dc:language", false)); mappings.put("license", new NameXPathMapping("dcterms.license","/pn:description/dcterms:license/@valueURI", false)); mappings.put("licenseText", new NameXPathMapping("NO_INDEX", "/pn:description/dcterms:license", false)); // mappings.put("location", new NameXPathMapping("dc.location","/pn:description/dc:location", false)); mappings.put("place", new NameXPathMapping("dcterms.spatial","/pn:description/dc:place", false, true)); // mappings.put("publisher", new NameXPathMapping("dc.publisher","/pn:description/dc:publisher", false)); mappings.put("rightsHolder", new NameXPathMapping("dcterms.rightsHolder","/pn:description/dcterms:rightsHolder", false)); mappings.put("spatial", new NameXPathMapping("dcterms.spatial", "/pn:description/dcterms:spatial", false, true)); mappings.put("spatial-format", new NameXPathMapping("NO_INDEX", "/pn:description/dcterms:spatial/@encSchemeURI", false, true)); mappings.put("subject", new NameXPathMapping("dc.subject","/pn:description/dc:subject", false)); mappings.put("temporal", new NameXPathMapping("dcterms.temporal", "/pn:description/dcterms:temporal", false)); mappings.put("thumbnail", new NameXPathMapping("pndsterms.thumbnail","/pn:description/pndsterms:thumbnail/@valueURI", false)); mappings.put("thumbnail2", new NameXPathMapping("pndsterms.thumbnail","/pn:description/pndsterms:thumbnail", false)); mappings.put("title", new NameXPathMapping("dc.title","/pn:description/dc:title", false)); mappings.put("type", new NameXPathMapping("dcmi.type","/pn:description/dc:type", false)); // mappings.put("materials", new NameXPathMapping("e20cl.materials", "/pn:description/e20cl:materials", false)); // mappings.put("size", new NameXPathMapping("e20cl.size", "/pn:description/e20cl:size", false)); // mappings.put("creditLine", new NameXPathMapping("e20cl.creditLine", "/pn:description/e20cl:creditLine", false)); // mappings.put("relatedSubject", new NameXPathMapping("e20cl.relatedSubject", "/pn:description/e20cl:relatedSubject", false)); mappings.put("subject.person", new NameXPathMapping("e20cl.relatedPerson", "/pn:description/e20cl:relatedPerson", false)); mappings.put("subject.organisation", new NameXPathMapping("e20cl.relatedOrganisation", "/pn:description/e20cl:relatedOrganisation", false)); mappings.put("subject.event", new NameXPathMapping("e20cl.relatedEvent", "/pn:description/e20cl:relatedEvent", false)); // mappings.put("relation", new NameXPathMapping("dc.relation", "/pn:description/dc:relation", false)); return mappings; } /** * Check through the data in the mappings (once filled with values) - if something is * required then check that it isn't null i.e. that we have found a value for it * @throws XPathExpressionException thrown if a field that is required has not been filled in from the XML document */ private void checkForRequiredData(HashMap xpathMappings) throws XPathExpressionException { // Loop through all of the entries and consider those where required == true for(String key: xpathMappings.keySet()) { NameXPathMapping thisEntry = xpathMappings.get(key); if ( thisEntry.isRequired() ) { // We care about it being a non-null value - check String[] value = thisEntry.getValues(); if ( value == null ) { log.debug("The required field: " + key + " doesn't have a value in the data file"); throw(new XPathExpressionException("Required field: " + key + " does not have a value in the data")); } } } } /** * Loop through the retrieved data and normalise it if appropriate * * @param xpathMappings The list of mappings and data to normalise * @return The normalised list of mappings and data */ private HashMap normaliseData(HashMap xpathMappings) { log.debug("In normaliseData"); Set keySet = xpathMappings.keySet(); log.debug("keySet size: " + keySet.size()); // Normalise the dcmi:type of the data if ( keySet.contains("type") ) { // There is a type entry, does it have a value / values? NameXPathMapping typeMapping = xpathMappings.get("type"); if ( typeMapping != null) { String[] types = xpathMappings.get("type").getValues(); if ( types == null ) { types = new String[1]; types[0] = "Unknown"; } List newTypes = new ArrayList(); for(int ctr = 0; ctr < types.length; ctr++) { String[] tempTypes = NormaliseTypes.normalise(types[ctr]); for(int ctr2 = 0; ctr2 < tempTypes.length; ctr2++) { newTypes.add(tempTypes[ctr2]); } } String[] newTypesAsArray = newTypes.toArray(new String[0]); // Remember the normalised values typeMapping.setValues(newTypesAsArray); xpathMappings.put("type", typeMapping); } } // Normalise the title - remove ^ from the string if ( keySet.contains("title") ) { // There is a title entry NameXPathMapping titleMapping = xpathMappings.get("title"); if ( titleMapping != null ) { String[] titles = titleMapping.getValues(); if ( titles == null || titles.length == 0 || titles[0] == null) { titles = new String[1]; titles[0] = "Unknown"; } log.debug("titles : " + titles + " length: " + titles.length); log.debug("titles[0]: " + titles[0]); if ( titles[0].contains("^") ) { titles[0] = titles[0].replaceAll("\\^", ""); } titleMapping.setValues(titles); xpathMappings.put("title", titleMapping); } } // Normalise the subjects - separate any values out by commas so that they // are indexed better if ( keySet.contains("subject") ) { // There is a subject entry NameXPathMapping subjectMapping = xpathMappings.get("subject"); if ( subjectMapping != null ) { String[] subjects = subjectMapping.getValues(); if ( subjects != null && subjects.length > 0 ) { // We have at least one subject to parse.. List parsedSubjectList = new ArrayList(); for (String thisSubject: subjects ) { if ( thisSubject != null ) { if ( thisSubject.contains(",") ) { // A comma separated list - separate and remember each value separately String[] separatedSubjects = thisSubject.split(","); for(String singleSubject: separatedSubjects) { parsedSubjectList.add(singleSubject); } } else { // No comma, so just remember the entire value parsedSubjectList.add(thisSubject); } } } // We now have all of the values separately - convert this into an array and then // remember it for later access String[] parsedArray = parsedSubjectList.toArray(new String[parsedSubjectList.size()]); if ( parsedArray != null && parsedArray.length > 0 ) { subjectMapping.setValues(parsedArray); xpathMappings.put("subject", subjectMapping); } } } } return xpathMappings; } /** * Get the value of the element or attribute given by the specified xpath expression * @param metadata_record The document to apply the xpath to * @param xpath The xpath expression to the element or attribute to consider * @param namespace_node The set of namespaces used in the xpath and document * @return The text contents of the elements or attributes as an array * @throws javax.xml.transform.TransformerException thrown if there's an error with the XPath */ protected static String[] getValues(Node metadata_record, String xpath, Node namespace_node) throws javax.xml.transform.TransformerException { // First work out whether we're looking for an attribute or a full element // - we're looking at an attribute if there is an @ after the last / // (and the expression doesn't end with ]. String node_xpath = xpath; String attribute_xpath = ""; if ( !xpath.endsWith("]") ) { int indexOfLastSlash = xpath.lastIndexOf("/"); int indexOfLastAt = xpath.lastIndexOf("@"); if ( indexOfLastAt > indexOfLastSlash ) { // Dealing with an attribute attribute_xpath = xpath.substring(indexOfLastAt + 1); node_xpath = xpath.substring(0, indexOfLastAt - 1); } } // Go and get the node(s) we're interested in NodeIterator nodeList = XPathAPI.selectNodeIterator(metadata_record, node_xpath, namespace_node); String[] returnValue = null; ArrayList tempList = new ArrayList(); Node actualNode; while ((actualNode = nodeList.nextNode()) != null) { // We've got a node - now are we getting an attribute or not? String value = null; if ( !"".equals(attribute_xpath) ) { value = ((Element)actualNode).getAttribute(attribute_xpath); } else { value = extractText(actualNode); } // Unescape any html entites, etc. in the string (a couple of times since there are some places // that encode multiple times and then remove the HTML // log.debug("about to unescape the value: " + value); if ( value != null ) { value = StringEscapeUtils.unescapeHtml(StringEscapeUtils.unescapeHtml(value)); value = value.replaceAll("\\<.*?\\>", ""); } // log.debug("value after unescaping and stripping tags: " + value); tempList.add(value); } returnValue = (String[])tempList.toArray(new String[tempList.size()]); return returnValue; } protected static String extractText(Node n) { try { Node node = XPathAPI.selectSingleNode(n,"./text()"); if ( node != null ) { node.normalize(); return node.getNodeValue().trim(); } } catch ( javax.xml.transform.TransformerException te ) { te.printStackTrace(); } return null; } }