package com.k_int.aggregator.dpp.timers; import com.k_int.aggregator.datamodel.AggregatorCollectionHDO; import com.k_int.aggregator.datamodel.AggregatorResourceHDO; import com.k_int.aggregator.datamodel.AggregatorSourceHDO; import com.k_int.aggregator.dpp.actions.admin.DTOReindexThread; import com.k_int.aggregator.dpp.timers.datamodel.CollectionUpdateTimerHDO; import com.k_int.aggregator.harvest.datamodel.HarvestInstructionHDO; import com.k_int.aggregator.repository.RepositoryStoreOptions; import com.k_int.aggregator.util.SOLRHelper; import com.k_int.discover.datamodel.CollLinkTypeEnum; import com.k_int.discover.datamodel.CultureGrid_AutomaticCollLinkHDO; import com.k_int.discover.datamodel.CultureGrid_CollectionAutomaticDataHDO; import com.k_int.discover.datamodel.CultureGrid_CollectionHDO; import com.k_int.discover.service.dtoHandler.DTODocumentHandlerFactory; import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.ModifiableSolrParams; import org.hibernate.*; import org.springframework.context.ApplicationContext; public class CollectionAutoUpdateJob implements Runnable { private SessionFactory sf; private CommonsHttpSolrServer solrServer; private DTODocumentHandlerFactory dtoHandlerFactory; private SOLRHelper solrHelper; private Log log = LogFactory.getLog(CollectionAutoUpdateJob.class); private final static String ALL_COLLS_QUERY = "select x from com.k_int.aggregator.datamodel.AggregatorCollectionHDO x"; private final static String RELATED_OAI_INSTR_QUERY = "select x from com.k_int.aggregator.harvest.datamodel.HarvestInstructionHDO x where x.parentCollection.id = ?"; private Set allowedUpdateIntervals = new HashSet(); public CollectionAutoUpdateJob(ApplicationContext ctx) { sf = (SessionFactory) ctx.getBean("AggregatorSessionFactory"); solrServer = (CommonsHttpSolrServer)ctx.getBean("CG-SOLRJ"); dtoHandlerFactory = (DTODocumentHandlerFactory)ctx.getBean("DTOHandlerFactory"); solrHelper = (SOLRHelper)ctx.getBean("SolrHelper"); allowedUpdateIntervals.add("MINUTE"); allowedUpdateIntervals.add("HOUR"); allowedUpdateIntervals.add("DAY"); } public void run() { log.debug("IN THE COLLECTION AUTO UPDATE JOB RUN METHOD"); Session sess = null; Transaction tx = null; // Read the relevant properties to get the next due interval, etc. ResourceBundle bundle = null; String updateIntervalUnit = "Day"; int updateIntervalVal = 1; try { bundle = ResourceBundle.getBundle("autoUpdate", Locale.getDefault()); if ( bundle.containsKey("com.k_int.dpp.auto.update.run.interval") ) updateIntervalVal = Integer.parseInt(bundle.getString("com.k_int.dpp.auto.update.run.interval")); if ( bundle.containsKey("com.k_int.dpp.auto.update.run.interval.unit") ) updateIntervalUnit = bundle.getString("com.k_int.dpp.auto.update.run.interval.unit"); if ( !allowedUpdateIntervals.contains(updateIntervalUnit.toUpperCase()) ) { // An update interval unit has been specified that isn't allowed - log it and reset to the default log.error("Auto update properties file includes an update interval unit value that isn't allowed, resetting to the default. Allowed values: minute, hour, day. Specified value: " + updateIntervalUnit); updateIntervalUnit = "Day"; } } catch (MissingResourceException mre) { // Doesn't matter as we'll stick with the defaults - log it though log.error("Unable to find the autoUpdate.properties file: " + mre.getMessage()); } try { sess = sf.openSession(); Set resourceIdsToReindex = new HashSet(); // Get all of the collections that we want to update Query q = sess.createQuery(ALL_COLLS_QUERY); @SuppressWarnings("unchecked") List allColls = (List)q.list(); // Loop through each collection and get the relevant information to update for(AggregatorCollectionHDO coll: allColls) { // Check to see if this collection is linked to a datamodel object that we can update if ( coll.getLinkedResource() != null ) { // Linked - update data log.debug("Found a collection that is linked to a datamodel object so going to do some updates. Collection id / code: " + coll.getId() + " / " + coll.getCollectionCode()); resourceIdsToReindex.add(coll.getLinkedResource().getId()); // Set up all of the information we want to store.. Set autoSubjects = new LinkedHashSet(); Long autoTotalNum = 0l; String autoAccrualStatus = ""; CultureGrid_AutomaticCollLinkHDO isPartOfData = null; CultureGrid_AutomaticCollLinkHDO linkedCollData = null; Set hasPartData = new LinkedHashSet(); Set associatedCollData = new LinkedHashSet(); // Go and get the top 50 subjects from this collection and the total number of records in the collection (and its subcollections) try { ModifiableSolrParams solrSearch = new ModifiableSolrParams(); solrSearch.set("q","dcterms.isPartOf:"+coll.getCollectionCode()); solrSearch.set("facet","true"); solrSearch.set("facet.field","dc.subject"); solrSearch.set("facet.minCount","1"); solrSearch.set("rows","0"); solrSearch.set("facet.limit","50"); QueryResponse response = solrServer.query(solrSearch); FacetField subjectData = response.getFacetField("dc.subject"); List subjects = subjectData.getValues(); for(Count aFacet: subjects) { String subjectValue = aFacet.getName(); Long subjectCount = aFacet.getCount(); autoSubjects.add(subjectValue); log.debug("Subject value: " + subjectValue + " subjectCount: " + subjectCount); } Long totalNum = response.getResults().getNumFound(); autoTotalNum = totalNum; log.debug("Num found: " + totalNum); } catch (SolrServerException sse) { log.error("SolrServerException thrown when setting up and searching for the top subjects for this collection: " + sse.getMessage()); sse.printStackTrace(); } // Go and work out whether we have an OAI instruction that feeds into this collection (or a parent) and use the // next due value for the instruction (if there is one) to decide whether we're ever going to get data from there again HarvestInstructionHDO relatedOaiInstr = findRelatedOAIInstruction(sess, coll); String accrualStatus = ""; if ( relatedOaiInstr != null ) { log.debug("Related harvest instruction found: " + relatedOaiInstr.getId()); // We have an OAI instruction - when is it next due? // Date suspendDate = new Date(3000-1900,0,1); // Year 3000.. Calendar suspendCalendar = new GregorianCalendar(); suspendCalendar.set(3000, 0, 1, 0, 0, 0); suspendCalendar.set(Calendar.MILLISECOND, 0); Date suspendDate = suspendCalendar.getTime(); if ( relatedOaiInstr.getStatus().getNextDue().getTime() == suspendDate.getTime() ) { // The instruction is suspended.. log.debug("Harvest instruction is suspended"); accrualStatus = "Accrual policy: Active; Status: OAI harvest; Periodicity: Closed"; // TODO } else { // The instruction is not suspended.. log.debug("Harvest instruction is active"); String periodicity = ""; accrualStatus = "Accrual policy: Active; Status: OAI harvest; Periodicity: " + periodicity; } } else { // No harvest instruction - work out the status. log.debug("No related harvest instruction found"); accrualStatus = "Accrual policy: Passive; Status: Upload; Periodicity: Irregular"; } autoAccrualStatus = accrualStatus; // Add in the cached linked coll information linkedCollData = CultureGrid_AutomaticCollLinkHDO.lookupOrCreate(sess, coll.getId().toString(), CollLinkTypeEnum.LINKED_COLL); linkedCollData.setCollTitle(coll.getName()); linkedCollData.setCollCode(coll.getCollectionCode()); log.debug("linkedColl data set to: " + linkedCollData.toString()); // Add in the cached isPartOf information AggregatorCollectionHDO parentColl = AggregatorCollectionHDO.lookupByChildCollectionCode(sess, coll.getCollectionCode()); if ( parentColl != null ) { log.debug("Parent collection found for isPartOf - setting up the values"); isPartOfData = CultureGrid_AutomaticCollLinkHDO.lookupOrCreate(sess, parentColl.getId().toString(), CollLinkTypeEnum.IS_PART); isPartOfData.setCollTitle(parentColl.getName());// Should this potentially come from the linked resource? isPartOfData.setCollCode(parentColl.getCollectionCode()); log.debug("isPartOf set to: " + isPartOfData.toString()); // TODO - what to get? } // Add in the cached hasPart information if ( coll.getChildrenCollections() != null && !coll.getChildrenCollections().isEmpty() ) { // Loop through each child and add the relevant information.. for(AggregatorCollectionHDO child: coll.getChildrenCollections()) { CultureGrid_AutomaticCollLinkHDO anHasPart = CultureGrid_AutomaticCollLinkHDO.lookupOrCreate(sess, child.getId().toString(), CollLinkTypeEnum.HAS_PART); anHasPart.setCollTitle(child.getName()); // Shold this come from the linked resource? anHasPart.setCollCode(child.getCollectionCode()); hasPartData.add(anHasPart); log.debug("Added a hasPart of: " + anHasPart.toString()); } } // Add in cached associated collections based on providers // First get the list of all providers associated with this collection List associatedProviders = AggregatorSourceHDO.lookupByCollection(sess, coll.getCollectionCode()); // Now loop through these providers and get all of the collections they are associated with (exluding this collection) Set allAssocColls = new HashSet(); for(AggregatorSourceHDO aProvider: associatedProviders) { if ( aProvider.getAssociatedCollections() != null && !aProvider.getAssociatedCollections().isEmpty() ) { for(AggregatorCollectionHDO assocColl: aProvider.getAssociatedCollections()) { // Only add in collections that aren't this one! if ( !assocColl.equals(coll) ) { allAssocColls.add(assocColl); } } } } for(AggregatorCollectionHDO assocColl: allAssocColls) { CultureGrid_AutomaticCollLinkHDO anAssoc = CultureGrid_AutomaticCollLinkHDO.lookupOrCreate(sess, assocColl.getId().toString(), CollLinkTypeEnum.ASSOCIATED); anAssoc.setCollTitle(assocColl.getName()); // Shold this come from the linked resource? anAssoc.setCollCode(assocColl.getCollectionCode()); associatedCollData.add(anAssoc); log.debug("Added an assoicated coll of: " + anAssoc.toString()); } // Now actually store the worked out automatic information back into the object AggregatorResourceHDO linkedRes = coll.getLinkedResource(); CultureGrid_CollectionHDO linkedColl = (CultureGrid_CollectionHDO)CultureGrid_CollectionHDO.lookupByResourceId(sess, linkedRes.getId()); if ( linkedColl != null ) { CultureGrid_CollectionAutomaticDataHDO autoData = linkedColl.getAutoData(); if ( autoData == null ) { autoData = new CultureGrid_CollectionAutomaticDataHDO(); tx = sess.beginTransaction(); linkedColl.setAutoData(autoData); sess.save(autoData); tx.commit(); } autoData.setSubjects(autoSubjects); autoData.setNumOfRecords(autoTotalNum); autoData.setAccrualStatus(autoAccrualStatus); autoData.setLinkedColl(null); autoData.setLinkedColl(linkedCollData); autoData.setIsPartOf(null); autoData.setIsPartOf(isPartOfData); autoData.setHasPart(null); autoData.setHasPart(hasPartData); autoData.setAssociatedColls(null); autoData.setAssociatedColls(associatedCollData); // Actually save the automatically updated information into the database tx = sess.beginTransaction(); sess.saveOrUpdate(isPartOfData); for(CultureGrid_AutomaticCollLinkHDO aLink: hasPartData) { sess.saveOrUpdate(aLink); } for(CultureGrid_AutomaticCollLinkHDO aLink: associatedCollData) { sess.saveOrUpdate(aLink); } sess.update(coll); tx.commit(); } else { log.error("Linked coll not found - problem!!"); } // Add in information about how the collection can be retrieved from CG - solr, oai, etc. // TODO ? } else { // Not linked - can't update log.debug("Found collection that's not linked to a datamodel object so not doing any updating.."); } } // Go and reindex the records that have possibly been updated to push the changes out to the various // places (solr, etc.) log.debug("About to reindex the relevant collections. Num to reindex: " + resourceIdsToReindex.size()); reindexRecords(resourceIdsToReindex); log.debug("Got to the end of the collection looping.."); // Update the fact that we've run this and set the next due to some point in the future Query timerQuery = sess.createQuery("Select x from com.k_int.aggregator.dpp.timers.datamodel.CollectionUpdateTimerHDO x"); log.debug("Just created the query.."); log.debug("About to run the query to get back the timer.."); CollectionUpdateTimerHDO timer = (CollectionUpdateTimerHDO)timerQuery.uniqueResult(); log.debug("About to work out the next due time and change it"); Calendar nextDueCal = new GregorianCalendar(); nextDueCal.setTime(timer.getNextDueTime()); if ( updateIntervalUnit.equalsIgnoreCase("Day") ) { // Add a number of days nextDueCal.add(Calendar.DAY_OF_WEEK, updateIntervalVal); } else if ( updateIntervalUnit.equalsIgnoreCase("Hour") ) { // Add a number of hours nextDueCal.add(Calendar.HOUR_OF_DAY, updateIntervalVal); } else { // Add a number of minutes nextDueCal.add(Calendar.MINUTE, updateIntervalVal); } log.debug("About to save the timer information.. getTime = " + nextDueCal.getTime()); tx = sess.beginTransaction(); timer.setLastRunTime(timer.getNextDueTime()); timer.setNextDueTime(nextDueCal.getTime()); sess.update(timer); tx.commit(); log.debug("Just set the next due time to be: " + timer.getNextDueTime()); } catch (HibernateException he) { log.error("HibernateException thrown when updating collections: " + he.getMessage()); he.printStackTrace(); } finally { if ( sess != null && sess.isOpen() ) { try { sess.close(); } catch (Exception e) { log.error("Exception thrown when closing session: " + e.getMessage()); e.printStackTrace(); } } } } /** * Work out whether there is an OAI harvest instruction in the system related to this collection or its ancestors * @param sess The session to work with * @param collToCheck The collection to check for * @return The first found related harvest instruction if there is one, null otherwise */ private HarvestInstructionHDO findRelatedOAIInstruction(Session sess, AggregatorCollectionHDO collToCheck) { HarvestInstructionHDO retval = null; if ( collToCheck != null ) { Query q = sess.createQuery(RELATED_OAI_INSTR_QUERY); q.setParameter(0, collToCheck.getId()); @SuppressWarnings("unchecked") List possibleInstructions = (List)q.list(); if ( possibleInstructions == null || possibleInstructions.isEmpty() ) { // No instruction found - look with this collection's parent collection if it has one AggregatorCollectionHDO parentColl = AggregatorCollectionHDO.lookupByChildCollectionCode(sess, collToCheck.getCollectionCode()); if ( parentColl != null ) { retval = findRelatedOAIInstruction(sess, parentColl); } } else { // We have at least one instruction - get the first one and return it retval = possibleInstructions.iterator().next(); } } return retval; } private void reindexRecords(Set resourceIds) { log.debug("In the reindex records method"); if ( resourceIds != null && !resourceIds.isEmpty() ) { log.debug("About to convert the set into an array.."); Object[] idToReindexAsObj = resourceIds.toArray(); Long[] idToReindex = new Long[idToReindexAsObj.length]; for(int ctr = 0; ctr < idToReindex.length; ctr++) { idToReindex[ctr] = (Long)idToReindexAsObj[ctr]; } log.debug("Got " + idToReindex.length + " records to reindex"); Long reindexOptions = RepositoryStoreOptions.REPO_STORE_OPTIONS_CACHE_THUMBNAIL | RepositoryStoreOptions.REPO_STORE_OPTIONS_DO_NOT_PERFORM_DATA_AUGMENTATION; DTOReindexThread reindexRunnable = new DTOReindexThread(idToReindex, reindexOptions, solrHelper, sf, "CollectionAutoUpdateReindexThread", dtoHandlerFactory); Thread reindexThread = new Thread(reindexRunnable); log.debug("About to start reindexing for collection automatic updates"); reindexThread.start(); try { reindexThread.join(); log.debug("Reindexing complete within collection automatic update job item.."); } catch(InterruptedException ie) { log.error("Interrupted exception thrown when attempting to join the collection automatic update job reindexing thread: " + ie.getMessage()); ie.printStackTrace(); // TODO - do something with this error... } // Clear up and free memory reindexThread = null; reindexRunnable = null; } log.debug("Finished reindexing.."); } }