package com.k_int.aggregator.dpp.actions; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import javax.servlet.http.HttpServletRequest; import org.apache.commons.io.FileUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.struts2.interceptor.ServletRequestAware; import org.hibernate.Session; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import com.k_int.aggregator.config.DPP_Properties; import com.k_int.aggregator.core.AggregatorService; import com.k_int.aggregator.datamodel.AggregatorCollectionHDO; import com.k_int.aggregator.datamodel.AggregatorSourceHDO; import com.k_int.peoples_record.repository.Repository; import com.k_int.discover.service.dtoHandler.DTODocumentHandlerFactory; import com.k_int.aggregator.dpp.util.DepositUtils; import com.k_int.aggregator.dpp.util.ProcessingUtils; import com.k_int.aggregator.dpp.util.SpreadsheetParseUtilImpl; import com.k_int.aggregator.dpp.util.UploadUtils; import com.k_int.aggregator.util.SOLRHelper; import com.k_int.discover.datamodel.dto.CultureGrid_BaseDTO; import com.k_int.svc.identity.service.IdentityService; import com.opensymphony.xwork2.ActionSupport; public class SpreadsheetUploadAction extends ActionSupport implements ServletRequestAware, ApplicationContextAware { private static final long serialVersionUID = -2342139193275287096L; protected HttpServletRequest request; protected ApplicationContext ctx; private org.hibernate.SessionFactory factory = null; private IdentityService identity_service; public void setServletRequest(HttpServletRequest request) { this.request = request; } public void setApplicationContext(ApplicationContext ctx) { this.ctx = ctx; } public void setSessionFactory(org.hibernate.SessionFactory factory) { this.factory = factory; } public org.hibernate.SessionFactory getSessionFactory() { return factory; } public void setIdentityService(IdentityService identity_service) { this.identity_service = identity_service; } private static Log log = LogFactory.getLog(SpreadsheetUploadAction.class); private Long resourceId; public void setResourceId(Long resourceId) { this.resourceId = resourceId; } public Long getResourceId() { return this.resourceId; } private int numUploaded = 0; private int numSuccess = 0; public int getNumUploaded() { return numUploaded; } public int getNumSuccess() { return numSuccess; } private File metadataUpload; // The actual metadata private String metadataUploadContentType; private String metadataUploadFileName; public File getMetadataUpload() { return metadataUpload; } public void setMetadataUpload(File metadataUpload) { this.metadataUpload = metadataUpload; } public String getMetadataUploadContentType() { return metadataUploadContentType; } public void setMetadataUploadContentType(String metadataUploadContentType) { this.metadataUploadContentType = metadataUploadContentType; } public String getMetadataUploadFileName() { return metadataUploadFileName; } public void setMetadataUploadFileName(String metadataUploadFileName) { this.metadataUploadFileName = metadataUploadFileName; } private String action; public void setHiddenActionInput(String action) { this.action = action; } private Map possibleCollections; private String collection = null; private String source = null; public Map getPossibleCollections() { return this.possibleCollections; } public String getCollection() { return this.collection; } public void setCollection(String collection) { this.collection = collection; } public void setSource(String source) { this.source = source; } public String getSource() { return this.source; } private String resultMessage; public String getResultMessage() { return this.resultMessage; } private String loggedInUsername = null; public String getLoggedInUsername() { return this.loggedInUsername; } private Repository repository; public void init() { log.debug("init method called in the UploadFileForResourceAction action"); this.repository = (Repository)ctx.getBean("ObjectRepository"); } public String execute() { String returnValue = SUCCESS; // Get the list of possible collections that the user can select Session sess = null; try { sess = factory.openSession(); List collections = AggregatorCollectionHDO.findCollsWithDepPrivs(sess, identity_service, request.getUserPrincipal().getName(), request.isUserInRole("SYSTEM.admin")); // Now setup our map of possible collections possibleCollections = new TreeMap(); for (AggregatorCollectionHDO collection : collections) { possibleCollections.put(collection.getName(), collection.getCollectionCode()); } if ( this.possibleCollections != null && this.collection == null ) { Set keyset = this.possibleCollections.keySet(); Iterator keysetIter = keyset.iterator(); if ( keysetIter.hasNext() ) { this.collection = this.possibleCollections.get(keysetIter.next()); log.debug("setting the collection to " + this.collection); } } if (possibleCollections == null) { log.error("No collection to upload into!"); returnValue = "uploadError"; resultMessage = "Unable to determine where to upload your records to. Please contact the system administrator to check that you have the required upload permissions"; } else { this.loggedInUsername = request.getUserPrincipal().getName(); if ( request.getMethod().equalsIgnoreCase("post") ) { if ( action != null && "upload".equalsIgnoreCase(action) ) { // Posting a file.. // Check the user has chosen a collection and provider if ( this.collection == null || this.source == null ) { log.error("No source or collection selected to upload into!"); returnValue = "uploadError"; resultMessage = "Unable to upload your records to without a selected collection or upload source. Please select a collection and releated source and try again."; } else { // Set up some required properties and retrieve various required beans // Read the relevant properties to get the next due interval, etc. String repoPublicDir = DPP_Properties.getRepositoryURL(); if ( repoPublicDir == null || repoPublicDir.trim().isEmpty() ) { // No public dir retrieved from config - complain and stop processing log.error("No public repository path retrieved from config - can't link to any uploaded files, etc. so not continuing with upload processing"); returnValue = ERROR; resultMessage = "Unable to identify the public path to any uploaded media files for image hosting, etc. Unable to continue processing your upload."; } else { SOLRHelper solrHelper = (SOLRHelper)ctx.getBean("SolrHelper"); log.info(" collection code:"+collection); log.info(" source identifier:" + source); log.info(" metadata contentType:"+metadataUploadContentType); log.info(" metadata filename:"+metadataUploadFileName); log.info(" metadata file:"+metadataUpload); // First load in the metadata file - needs to be .xml.. if ( metadataUpload != null ) { try { InputStream is = metadataUpload.toURI().toURL().openStream(); if ( is != null ) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int b = is.read(); while ( b != -1 ) { baos.write(b); b = is.read(); } byte[] fileAsArray = baos.toByteArray(); baos.close(); is.close(); is = null; baos = null; // Handle the processing of a zip (or other) archive File metadataFile = null; String type = metadataUploadContentType; String metadataType = "unknown"; // First work out where to extract (if required) to and create the required directories String containingDir = System.getProperty("java.io.tmpdir"); String extractDirPath = containingDir + File.separatorChar + "uploadArea" + File.separatorChar; File extractDir = new File(extractDirPath); if ( !extractDir.exists() ) { log.debug("Parent extraction directory doesn't exist - creating it at " + extractDirPath); extractDir.mkdirs(); } // Now work out what the directory will be for this particular extraction File tempDir = File.createTempFile("extract", Long.toString(System.nanoTime()), extractDir); if ( tempDir.exists() ) { tempDir.delete(); } tempDir.mkdir(); String pathToAttachments = tempDir.getPath(); try { log.debug("Actual extraction directory created at " + tempDir.getPath()); // TODO - need to remember to remove the archive at the end.. if ( type.equals("application/zip") || type.equals("application/x-zip") || type.equals("application/x-gzip") || type.equals("application/x-zip-compressed") || type.equals("application/octet-stream") || type.equals("application/x-compress") || type.equals("application/x-compressed") || type.equals("multipart/x-zip") ) { // We're dealing with a zip file of some sort that we need to unpack UploadUtils.unpack(fileAsArray, pathToAttachments); // Now read the metadata spreadsheet in the archive into the byte[] for continued processing // TODO - change this to work when we can also have .xml metadata.... String metadataFilePath = pathToAttachments + File.separatorChar + "metadata.xls"; metadataFile = new File(metadataFilePath); if ( !metadataFile.exists() ) { // .xls doesn't exist - try .xlsx instead metadataFilePath = pathToAttachments + File.separatorChar + "metadata.xlsx"; metadataFile = new File(metadataFilePath); if ( !metadataFile.exists() ) { // .xslx doesn't exist either - try .xml metadataFilePath = pathToAttachments + File.separatorChar + "metadata.xml"; metadataFile = new File(metadataFilePath); if ( metadataFile.exists() ) { metadataType = "xml"; } } else { metadataType = "xlsx"; } } else { metadataType = "xls"; } if ( metadataFile.exists() ) { // File exists - read it in.. InputStream metadataIs = metadataFile.toURI().toURL().openStream(); if ( metadataIs != null ) { ByteArrayOutputStream metadataBaos = new ByteArrayOutputStream(); int readByte = metadataIs.read(); while ( readByte != -1 ) { metadataBaos.write(readByte); readByte = metadataIs.read(); } // Replace the read in file with this so the rest of the code doesn't need to care // if we're working with a zip archive or not fileAsArray = metadataBaos.toByteArray(); metadataBaos.close(); metadataIs.close(); metadataIs = null; metadataBaos = null; } } else { // File doesn't exist.. (in any expected type) log.error("Unable to find metadata.xls, metadata.xlsx or metadata.xml in the uploaded archive!"); // TODO } } else { String lowerType = metadataUploadContentType.toLowerCase(); if ( lowerType.contains("excel") || ( !lowerType.contains("opendocument") && lowerType.contains("spreadsheet") ) ) { metadataType = "xls"; // Or xlsx - doesn't matter for this code though } else if ( lowerType.contains("xml") ) { metadataType = "xml"; } else { metadataType = "unknown"; } log.debug("Not dealing with an archive - still trying to work out the mime type though... metadataType decided to be: " + metadataType); } // Set up variables that are shared across the different types of parsing below AggregatorService aggregator = (AggregatorService) ctx.getBean("AggregatorService"); DTODocumentHandlerFactory dtoFactory = (DTODocumentHandlerFactory)ctx.getBean("DTOHandlerFactory"); // Depending on the type of uploaded object do the required type of processing if ( "xls".equals(metadataType) || "xlsx".equals(metadataType) ) { // We're dealing with a spreadsheet SpreadsheetParseUtilImpl parser = new SpreadsheetParseUtilImpl(); LinkedHashMap records = parser.parseCGSpreadsheet(fileAsArray); log.debug(records.size() + " records returned from parsing the spreadsheet"); // Loop through the returned records and deposit them one by one AggregatorCollectionHDO collHDO = AggregatorCollectionHDO.lookup(sess, collection); if ( collHDO == null ) { log.debug("collHDO is null"); } AggregatorSourceHDO sourceHDO = AggregatorSourceHDO.lookup(sess, source); if ( sourceHDO == null ) { log.debug("sourceHDO is null"); } log.debug("About to call depositSetOfRecords"); List resourceIds = DepositUtils.depositSetOfRecords(aggregator, repository, sess, collHDO, sourceHDO, request.getUserPrincipal().getName(), records, pathToAttachments, repoPublicDir); for(Long resId: resourceIds) { log.debug("deposit performed - returned ID: " + resId); } // Now index the created resources // Now everything is uploaded / stored in the DB pass the object through the handlers to index them into solr, etc. // Reprocess the resource to create different versions, index it, etc. log.debug("About to perform the reindexing of the record to generate the other artifacts"); int numOK = ProcessingUtils.reindexFollowingEdit(factory, dtoFactory, resourceIds, solrHelper); returnValue = "multiUploadSuccess"; this.numUploaded = records.size(); this.numSuccess = numOK; if (resourceIds.size() > 0) { this.resourceId = resourceIds.get(resourceIds.size() - 1); } // TODO - need to return more information... } else { // We're dealing with an unknown type! log.error("Unknown file type provided when attempting to upload metadata! metadataContentType = " + metadataUploadContentType); returnValue = ERROR; resultMessage = "Unknown file type provided when attempting to upload metadata. Currently supported formats are Excel spreadsheets (Office 97 - 2007)"; } } finally { // Delete the directory we created log.debug("Deleting temporary directory: " + pathToAttachments); FileUtils.deleteQuietly(tempDir); } // TODO - check that the user has the permissions to upload to the specified provider / collection } } catch ( MalformedURLException mue ) { log.error("MalformedURLException thrown when reading in the uploaded metadata file: " + mue.getMessage()); resultMessage = "Unable to upload the metadata record or one of its attachments. Please try again"; returnValue = ERROR; } catch ( IOException ioe ) { log.error("IOException thrown when reading in the uploaded metadata file: " + ioe.getMessage()); resultMessage = "Unable to read the uploaded metadata record or one of its attachments. If uploading a zip archive is it in the correct format?"; returnValue = ERROR; } } else { log.error("metadataUpload is null!"); addFieldError("metadataUpload", "A metadata record must be specified"); } } } } else { log.debug("action is :" + action); } } } } finally { if ( sess != null && sess.isOpen() ) { try { sess.close(); } catch (Exception e) {} } } return returnValue; } }