package com.k_int.aggr2.mimsy.data.util;


import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.HibernateException;
import org.hibernate.Query;
import org.hibernate.Session;

import com.k_int.aggr2.mimsy.data.AttributionDTO;
import com.k_int.aggr2.mimsy.data.CIIMContextDataDTO;
import com.k_int.aggr2.mimsy.data.CIIMGroupDataDTO;
import com.k_int.aggr2.mimsy.data.CIIMMasterDataDTO;
import com.k_int.aggr2.mimsy.data.ControlledReferenceDTO;
import com.k_int.aggr2.mimsy.data.DescriptionDTO;
import com.k_int.aggr2.mimsy.data.ExcavationDTO;
import com.k_int.aggr2.mimsy.data.ExtensionFieldDTO;
import com.k_int.aggr2.mimsy.data.GroupHierarchyDTO;
import com.k_int.aggr2.mimsy.data.MimsyCuratedGroupDTO;
import com.k_int.aggr2.mimsy.data.DataContextDTO;
import com.k_int.aggr2.mimsy.data.InstructionDTO;
import com.k_int.aggr2.mimsy.data.MimsyAgentDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyDateDTO;
import com.k_int.aggr2.mimsy.data.MimsyEventDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyMediaDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyObjectDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyPlaceDataDTO;
import com.k_int.aggr2.mimsy.data.MimsyPublicationDataDTO;
import com.k_int.aggr2.mimsy.data.MimsySiteDataDTO;
import com.k_int.aggr2.mimsy.data.MimsySubjectDataDTO;
import com.k_int.aggr2.mimsy.data.NameDTO;
import com.k_int.aggr2.mimsy.data.PlaceVariationDTO;
import com.k_int.aggr2.mimsy.data.RightsDTO;
import com.k_int.aggr2.mimsy.data.TextTypePairDTO;
import com.k_int.aggr2.mimsy.data.TitleDTO;
import com.k_int.aggr2.mimsy.data.hdo.AgentReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.AttributionHDO;
import com.k_int.aggr2.mimsy.data.hdo.CIIMDataContextHDO;
import com.k_int.aggr2.mimsy.data.hdo.CIIMDataContextMetadataHDO;
import com.k_int.aggr2.mimsy.data.hdo.CIIMGroupHDO;
import com.k_int.aggr2.mimsy.data.hdo.CIIMMediaDescriptionHDO;
import com.k_int.aggr2.mimsy.data.hdo.CollectorReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.DescriptionHDO;
import com.k_int.aggr2.mimsy.data.hdo.EventReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.ExcavationHDO;
import com.k_int.aggr2.mimsy.data.hdo.GroupHierarchyXRTreeHDO;
import com.k_int.aggr2.mimsy.data.hdo.InstructionHDO;
import com.k_int.aggr2.mimsy.data.hdo.LocationHDO;
import com.k_int.aggr2.mimsy.data.hdo.MakerReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.MediaReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyAgentDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyCuratedGroupHDO;
import com.k_int.aggr2.mimsy.data.hdo.BaseMimsyDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyDateHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyEventDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyMediaDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyObjectDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyPlaceDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsyPublicationDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsySiteDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.MimsySubjectDataHDO;
import com.k_int.aggr2.mimsy.data.hdo.NameHDO;
import com.k_int.aggr2.mimsy.data.hdo.ObjectReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.PlaceReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.PlaceVariationHDO;
import com.k_int.aggr2.mimsy.data.hdo.PublicationReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.RightsHDO;
import com.k_int.aggr2.mimsy.data.hdo.SchemaElementHDO;
import com.k_int.aggr2.mimsy.data.hdo.SchemaOptionValueHDO;
import com.k_int.aggr2.mimsy.data.hdo.SiteReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.StringReplacementHDO;
import com.k_int.aggr2.mimsy.data.hdo.SubjectHeadingHDO;
import com.k_int.aggr2.mimsy.data.hdo.SubjectReferenceHDO;
import com.k_int.aggr2.mimsy.data.hdo.TextTypePairHDO;
import com.k_int.aggr2.mimsy.data.hdo.TitleHDO;
import com.k_int.mimsy.ref.DataException;
import com.k_int.mimsy.ref.DataStatusEnum;
import com.k_int.mimsy.ref.GroupEditorialStatusEnum;
import com.k_int.mimsy.ref.OriginatorEnum;
import com.k_int.mimsy.ref.GroupPublicationStatusEnum;
import com.k_int.mimsy.ref.MimsyDataTypeEnum;
import com.k_int.mimsy.ref.ObjectConformanceEnum;
import com.k_int.mimsy.ref.TopGroupEditorialStatusEnum;

/**
 * 				
 * @author rob
 *				The main (and too large) class for converting between DTO's and HDO's
 */
public class DataManager 
{	
	public static Log log = LogFactory.getLog(DataManager.class);
	
	private static void copyInstructions(MimsyDataDTO dto, BaseMimsyDataHDO hdo)
	{
		Iterator<InstructionHDO> instr = hdo.getInstructions().iterator();
		while(instr.hasNext())
		{
			InstructionHDO inst = instr.next();
			InstructionDTO inst_dto = new InstructionDTO();
			inst_dto.setDate(inst.getDate());
			inst_dto.setInstruction(inst.getInstruction());
			inst_dto.setName(inst.getName());
			inst_dto.setStatus(inst.getStatus());
			inst_dto.setType(inst.getType());
			dto.getInstructions().add(inst_dto);
		}
		
	}
	private static MimsyMediaDataDTO convert(MimsyMediaDataHDO hdo)
	{
		MimsyMediaDataDTO retval = new MimsyMediaDataDTO(hdo.getMimsyIdentifier(), hdo.getLastUpdated(), hdo.getOriginator());
		retval.setCabinetImageNumber(hdo.getCabinetImageNumber());
		retval.setCaption(hdo.getCaption());
		retval.setChecksum(hdo.getChecksum());
		retval.setConformance(hdo.getConformance());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDonorFilename(hdo.getDonorFilename());
		retval.setDonorLocation(hdo.getDonorLocation());
		retval.setDonorRelativePath(hdo.getDonorRelativePath());
		retval.setFormat(hdo.getFormat());
		retval.setLargeDonorLocation(hdo.getLargeDonorLocation());
		retval.setLargeDonorRelativePath(hdo.getLargeDonorRelativePath());
		retval.setLargeFileLocation(hdo.getLargeFileLocation());
		retval.setLargeImagetype(hdo.getLargeImageType());
		retval.setLargeRelativePath(hdo.getLargeRelativePath());
		retval.setMediaType(hdo.getMediaType());
		retval.setMedium(hdo.getMedium());
		retval.setMidFileLocation(hdo.getMidFileLocation());
		retval.setMidRelativePath(hdo.getMidRelativePath());
		retval.setPreviewFileLocation(hdo.getPreviewFileLocation());
		retval.setPreviewRelativePath(hdo.getPreviewRelativePath());
		retval.setRecordType(hdo.getRecordType());
		retval.setRelationship(hdo.getRelationship());
		Iterator<RightsHDO> i = hdo.getRights().iterator();
		while(i.hasNext())
		{
			RightsHDO r_hdo = i.next();
			RightsDTO rights = new RightsDTO();
			rights.setCreditLine(r_hdo.getCreditLine());
			rights.setDetails(r_hdo.getDetails());
			rights.setFromDate(r_hdo.getFromDate());
			rights.setHolder(r_hdo.getHolder());
			rights.setToDate(r_hdo.getToDate());
			rights.setType(r_hdo.getType());
			retval.getRights().add(rights);
		}
		copyInstructions(retval,hdo);
		return retval;
	}
	
	 // status was not set so was change on 22.3.11
	private static DataContextDTO convert(Session sess,CIIMDataContextHDO hdo)
	{
		DataContextDTO retval = new DataContextDTO(hdo.getMimsyIdentifier(), hdo.getMaster().getMimsyIdentifier(), hdo.getGroup().getMimsyIdentifier(), hdo.getLastUpdated(), hdo.getDataType(), hdo.getOriginator());
		retval.setChecksum(hdo.getChecksum());
		retval.setConformance(hdo.getConformance());
		retval.setDataStatus(hdo.getDataStatus());
		CIIMMasterDataDTO master = new CIIMMasterDataDTO(toDataDTO(sess,hdo.getMaster()));
		retval.setMasterData(master);
		
		if(hdo.getCIIMMetadata()!=null)
		{
			CIIMContextDataDTO ciim_data = new CIIMContextDataDTO();
			ciim_data.setBoostValue(hdo.getCIIMMetadata().getBoost());
			ciim_data.setFeatured(hdo.getCIIMMetadata().getFeatured());
			
			 /* extract extension schema data */
			 Iterator<SchemaElementHDO> iter = hdo.getCIIMMetadata().getOptions().keySet().iterator();
			 
			 while(iter.hasNext())
			 {
				 SchemaElementHDO key = iter.next();
				 SchemaOptionValueHDO schema_opt_value_hdo = hdo.getCIIMMetadata().getOptions().get(key);
				 
				 if(schema_opt_value_hdo != null && key.getDefinition() != null)
				 {
					 ExtensionFieldDTO extension_field = new ExtensionFieldDTO(key.getDefinition().getName(), key.getDefinition().getSolrIdentifier(), schema_opt_value_hdo.getValues());
					 ciim_data.getExtensionFields().add(extension_field);
				 }	 
			 }
			
			retval.setCIIMData(ciim_data);
		}
		return retval;
	}
	
	private static MimsyAgentDataDTO convert(MimsyAgentDataHDO hdo)
	{
		MimsyAgentDataDTO retval = new MimsyAgentDataDTO(hdo.getMimsyIdentifier(), hdo.getLastUpdated(), hdo.getConformance());
		retval.setBirthDate(hdo.getBirthDate());
		retval.setBirthPlace(hdo.getBirthPlace());
		retval.setBriefBio(hdo.getBriefBio());
		retval.setCauseOfDeath(hdo.getCauseOfDeath());
		retval.setChecksum(hdo.getChecksum());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDeathDate(hdo.getDeathDate());
		retval.setDeathPlace(hdo.getDeathPlace());
		retval.setDeceased(hdo.isDeceased());
		retval.setDescription(hdo.getDescription());
		retval.setFirstName(hdo.getFirstName());
		retval.setGender(hdo.getGender());
		retval.setHonorarySuffix(hdo.getHonorarySuffix());
		retval.setIndividual(hdo.isIndividual());
		retval.setLastName(hdo.getLastName());
		retval.setNationality(hdo.getNationality());
		retval.setOccupation(hdo.getOccupation());
		retval.setPreferredName(hdo.getPreferredName());
		retval.setSuffix(hdo.getSuffix());
		retval.setTitlePrefix(hdo.getTitlePrefix());
		copyInstructions(retval,hdo);
		return retval;
	}
	
	private static MimsyEventDataDTO convert(MimsyEventDataHDO hdo)
	{
		MimsyEventDataDTO retval = new MimsyEventDataDTO(hdo.getMimsyIdentifier(),hdo.getLastUpdated(),hdo.getConformance());
		retval.setBriefName(hdo.getBriefName());
		retval.setCandidate(hdo.getCandidate());
		retval.setChecksum(hdo.getChecksum());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDescription(hdo.getDescription());
		retval.setDisplayDate(hdo.getDisplayDate());
		retval.setDuration(hdo.getDuration());
		retval.setEndDate(hdo.getEndDate());
		retval.setFullName(hdo.getFullName());
		retval.setScopeNote(hdo.getScopeNote());
		retval.setStartDate(hdo.getStartDate());
		retval.setUnits(hdo.getUnits());
		copyInstructions(retval,hdo);
		return retval;
	}
	
	private static MimsyPlaceDataDTO convert(MimsyPlaceDataHDO hdo)
	{
		MimsyPlaceDataDTO retval = new MimsyPlaceDataDTO(hdo.getMimsyIdentifier(),hdo.getLastUpdated(),hdo.getConformance());
		retval.setBroaderPlace(hdo.getBroaderPlace());
		retval.setChecksum(hdo.getChecksum());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDescription(hdo.getDescription());
		retval.setLevelNumber(hdo.getLevelNumber());
		retval.setNote(hdo.getNote());
		retval.setPlace(hdo.getPlace());
		retval.setPlaceType(hdo.getPlaceType());
		copyInstructions(retval,hdo);
		return retval;
	}
	
	private static MimsyPublicationDataDTO convert(MimsyPublicationDataHDO hdo)
	{
		MimsyPublicationDataDTO retval = new MimsyPublicationDataDTO(hdo.getMimsyIdentifier(),hdo.getLastUpdated(),hdo.getConformance());
		retval.setAuthor(hdo.getAuthor());
		retval.setBriefTitle(hdo.getBriefTitle());
		retval.setChecksum(hdo.getChecksum());
		retval.setCitation(hdo.getCitation());
		retval.setConformance(hdo.getConformance());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDescription(hdo.getDescription());
		retval.setEdition(hdo.getEdition());
		retval.setEditor(hdo.getEditor());
		retval.setIllustrated(hdo.getIllustrated());
		copyInstructions(retval,hdo);
		retval.setIsbnIssn(hdo.getIsbnIssn());
		retval.setIssue(hdo.getIssue());
		retval.setLastUpdated(hdo.getLastUpdated());
		retval.setLocation(hdo.getLocation());
		retval.setPagesTracks(hdo.getPagesTracks());
		retval.setPlacePublished(hdo.getPlacePublished());
		retval.setPublicationDate(hdo.getPublicationDate());
		retval.setPublicationType(hdo.getPublicationType());
		retval.setPublished(hdo.isPublished());
		retval.setPublisher(hdo.getPublisher());
		retval.setSeriesTitle(hdo.getSeriesTitle());
		Iterator<SubjectHeadingHDO> subjects = hdo.getSubjectHeadings().iterator();
		while(subjects.hasNext())
		{
			SubjectHeadingHDO subject = subjects.next();
			retval.getSubjectHeadings().add(subject.getSubjectHeading());
		}
		retval.setTitle(hdo.getTitle());
		retval.setTitleStatement(hdo.getTitleStatement());
		retval.setVolume(hdo.getVolume());
		retval.setVolumes(hdo.getVolumes());
		retval.setYearPublished(hdo.getYearPublished());
		
		return retval;
	}
	
	
	private static MimsySiteDataDTO convert(MimsySiteDataHDO hdo)
	{
		MimsySiteDataDTO retval = new MimsySiteDataDTO(hdo.getMimsyIdentifier(),hdo.getLastUpdated(), hdo.getConformance(),hdo.getSiteId());
		retval.setBorough(hdo.getBorough());
		retval.setChecksum(hdo.getChecksum());
		hdo.setConformance(hdo.getConformance());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDate(hdo.getDate());
		retval.setEastings(hdo.getEastings());
		Iterator<ExcavationHDO> i = hdo.getExcavations().iterator();
		while(i.hasNext())
		{
			ExcavationHDO e_hdo = i.next();
			ExcavationDTO e_dto = new ExcavationDTO();
			e_dto.setEnd(e_hdo.getEnd());
			e_dto.setOrganisation(e_hdo.getOrganisation());
			e_dto.setRole(e_hdo.getRole());
			e_dto.setStart(e_hdo.getStart());
			retval.getExcavations().add(e_dto);
		}
		copyInstructions(retval,hdo);
		retval.setLastUpdated(hdo.getLastUpdated());
		retval.setLocation(hdo.getLocation());
		retval.setName(hdo.getName());
		retval.setNorthings(hdo.getNorthings());
		retval.setOrganisation(hdo.getOrganisation());
		retval.setSiteType(hdo.getSiteType());
		retval.setSummary(hdo.getSummary());
		retval.setSiteYear(hdo.getSiteYear());		
	
		return retval;
	}
	
	private static MimsySubjectDataDTO convert(MimsySubjectDataHDO hdo)
	{
		MimsySubjectDataDTO retval = new MimsySubjectDataDTO(hdo.getMimsyIdentifier(),hdo.getLastUpdated(), hdo.getConformance());
		retval.setArea(hdo.getArea());
		retval.setCategory(hdo.getCategory());
		retval.setChecksum(hdo.getChecksum());
		retval.setConformance(hdo.getConformance());
		retval.setDataStatus(hdo.getDataStatus());
		retval.setDescription(hdo.getDescription());
		copyInstructions(retval, hdo);
		retval.setLastUpdated(hdo.getLastUpdated());
		retval.setPeriod(hdo.getPeriod());
		retval.setSubCategory(hdo.getSubCategory());
		retval.setSubject(hdo.getSubject());
		return retval;
		
	}
	
	private static MimsyObjectDataDTO convert(MimsyObjectDataHDO hdo)
	{
		//TODO add context
		MimsyObjectDataDTO retval = new MimsyObjectDataDTO(hdo.getMimsyIdentifier(), hdo.getObjectId(), hdo.getConformance(), hdo.getLastUpdated());
		retval.setAcquisition(hdo.getAcquisition());
		retval.setAdditionalInformation(hdo.getAdditionalInformation());
		Iterator<StringReplacementHDO> strings = hdo.getBoroughs().iterator();
		while(strings.hasNext())
		{
			retval.getBoroughs().add(strings.next().getValue());
		}
		retval.setChecksum(hdo.getChecksum());
		Iterator<MimsyDateHDO> dates = hdo.getCollectedDates().iterator();
		while(dates.hasNext())
		{
			MimsyDateHDO date = dates.next();
			MimsyDateDTO date_h = new MimsyDateDTO(date.getDateText(), date.getPrior());
			if(date.getAttribution()!=null)
			{
				AttributionHDO attrib = date.getAttribution();
				AttributionDTO a_h = new AttributionDTO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
				date_h.setAttribution(a_h);
			}
			date_h.setEarliest(date.getEarliest());
			date_h.setEra(date.getEra());
			date_h.setLatest(date.getLatest());
			date_h.setMethod(date.getMethod());
			date_h.setRelation(date.getRelation());
			retval.getCollectedDates().add(date_h);
		}
		//strings = hdo.getCollectorNames().iterator();
		//while(strings.hasNext())
		//{
		//	retval.getCollectorNames().add(strings.next().getValue());
		//}
		
		retval.setConservationAction(hdo.getConservationAction());
		retval.setCreditLine(hdo.getCreditLine());
		LocationHDO loc = hdo.getCurrentLocation();
		if(loc!=null)
		{
			retval.setCurrentLocation(loc.getCode());
			retval.setCurrentLocationDescription(loc.getName());
		}
		retval.setDataStatus(hdo.getDataStatus());
		
		Iterator<DescriptionHDO> descs = hdo.getDescriptions().iterator();
		while(descs.hasNext())
		{
			DescriptionHDO desc = descs.next();
			DescriptionDTO d_hdo = new DescriptionDTO(desc.getDescription(), desc.getType(), desc.getPrior());
			d_hdo.setDate(desc.getDate());
			d_hdo.setSource(desc.getSource());
			retval.getDescriptions().add(d_hdo);
			
		}
		retval.setDisplayDate(hdo.getDisplayDate());
		Iterator<NameHDO> names = hdo.getFunctionalClassification().iterator();
		while(names.hasNext())
		{
			NameHDO name = names.next();
			NameDTO name_h = new NameDTO(name.getName(), name.getType(), name.getPrevious());
		
			retval.getFunctionalClassification().add(name_h);
		}
		retval.setIntendedUse(hdo.getIntendedUse());
		retval.setItemCount(hdo.getItemCount());
		Iterator<AgentReferenceHDO> a_refs=  hdo.getLinkedAgents().iterator();
		while(a_refs.hasNext())
		{
			AgentReferenceHDO ref = a_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedAgents().add(dto);
		}
		
		Iterator<CollectorReferenceHDO> c_refs=  hdo.getLinkedCollectors().iterator();
		while(c_refs.hasNext())
		{
			CollectorReferenceHDO ref = c_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedCollectors().add(dto);
		}
		
		Iterator<EventReferenceHDO> e_refs=  hdo.getLinkedEvents().iterator();
		while(e_refs.hasNext())
		{
			EventReferenceHDO ref = e_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedEvents().add(dto);
		}
		
		Iterator<MakerReferenceHDO> m_refs=  hdo.getLinkedMakers().iterator();
		while(m_refs.hasNext())
		{
			MakerReferenceHDO ref = m_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedMakers().add(dto);
		}
		
		Iterator<MediaReferenceHDO> media_i = hdo.getLinkedMedia().iterator();
		while(media_i.hasNext())
		{
			MediaReferenceHDO media = media_i.next();
			
			MimsyMediaDataDTO media_dto = convert(media.getReference());
			retval.getLinkedMedia().put(media.getPosition(), media_dto);
		}
		
		
		Iterator<ObjectReferenceHDO> o_refs=  hdo.getLinkedObjects().iterator();
		while(o_refs.hasNext())
		{
			ObjectReferenceHDO ref = o_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedObjects().add(dto);
		}
		
		
		Iterator<PlaceReferenceHDO> p_refs=  hdo.getLinkedPlaces().iterator();
		while(p_refs.hasNext())
		{
			PlaceReferenceHDO ref = p_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			dto.setText(ref.getReference().getPlace());
			retval.getLinkedPlaces().add(dto);
		}
		
		Iterator<PublicationReferenceHDO> pub_refs=  hdo.getLinkedPublications().iterator();
		while(pub_refs.hasNext())
		{
			PublicationReferenceHDO ref = pub_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedPublications().add(dto);
		}
		
		Iterator<SiteReferenceHDO> s_refs=  hdo.getLinkedSites().iterator();
		while(s_refs.hasNext())
		{
			SiteReferenceHDO ref = s_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedSites().add(dto);
		}
		
		Iterator<SubjectReferenceHDO> sub_refs=  hdo.getLinkedSubjects().iterator();
		while(sub_refs.hasNext())
		{
			SubjectReferenceHDO ref = sub_refs.next();
			ControlledReferenceDTO dto = new ControlledReferenceDTO(ref.getReference().getMimsyIdentifier(), null, ref.getRelationship());
			retval.getLinkedSubjects().add(dto);
		}
		
		
		retval.setLoanAllowed(hdo.isLoanAllowed());
		dates = hdo.getMadeDates().iterator();
		while(dates.hasNext())
		{
			MimsyDateHDO date = dates.next();
			//if(date==null)
			//	continue; // TODO look at how this happens
			MimsyDateDTO date_h = new MimsyDateDTO(date.getDateText(), date.getPrior());
			if(date.getAttribution()!=null)
			{
				AttributionHDO attrib = date.getAttribution();
				AttributionDTO a_h = new AttributionDTO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
				date_h.setAttribution(a_h);
			}
			date_h.setEarliest(date.getEarliest());
			date_h.setEra(date.getEra());
			date_h.setLatest(date.getLatest());
			date_h.setMethod(date.getMethod());
			date_h.setRelation(date.getRelation());
			retval.getMadeDates().add(date_h);
		}
		
		//strings = hdo.getMakerNames().iterator();
		//while(strings.hasNext())
		//{
		//	retval.getMakerNames().add(strings.next().getValue());
		//}
		
		strings = hdo.getMaterials().iterator();
		while(strings.hasNext())
		{
			retval.getMaterials().add(strings.next().getValue());
		}
	
		retval.setMeasurements(hdo.getMeasurements());
				
		names = hdo.getNames().iterator();
		while(names.hasNext())
		{
			NameHDO name = names.next();
			NameDTO name_h = new NameDTO(name.getName(), name.getType(), name.getPrevious());
		
			retval.getNames().add(name_h);
		}
		retval.setNote(hdo.getNote());
		Iterator<TextTypePairHDO> pairs = hdo.getOtherNumbers().iterator();
		while(pairs.hasNext())
		{
			TextTypePairHDO pair = pairs.next();
			TextTypePairDTO h = new TextTypePairDTO(pair.getText(), pair.getType());
			retval.getOtherNumbers().add(h);
		}
		Iterator<TitleHDO> titles = hdo.getOtherTitles().iterator();
		while(titles.hasNext())
		{
			TitleHDO title 		= titles.next();
			AttributionDTO a_h	= null;
			if(title.getAttribution()!=null)
			{
				AttributionHDO attrib = title.getAttribution();
				a_h = new AttributionDTO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
			}
			TitleDTO h = new TitleDTO(title.getTitle(), title.getType(), title.getPrior(), a_h);
			retval.getOtherTitles().add(h);
		}
		retval.setOwnerStatus(hdo.getOwnerStatus());
		retval.setPagination(hdo.getPagination());
		retval.setPermanentLocation(hdo.getPermanentLocation());
		retval.setPlaceCollected(hdo.getPlaceCollected());
		retval.setPlaceMade(hdo.getPlaceMade());
		if(hdo.getPrimaryMaterial()!=null)
			retval.setPrimaryMaterial(hdo.getPrimaryMaterial().getValue());
		strings = hdo.getProjectCodes().iterator();
		while(strings.hasNext())
		{
			retval.getProjectCodes().add(strings.next().getValue());
		}
		retval.setProjectNotes(hdo.getProjectNotes());
		retval.setPublisher(hdo.getPublisher());
		retval.setRecordType(hdo.getRecordType());
		
		RightsHDO r_hdo  = hdo.getRights();
		if(r_hdo!=null)
		{
			RightsDTO rights = new RightsDTO();
			rights.setCreditLine(r_hdo.getCreditLine());
			rights.setDetails(r_hdo.getDetails());
			rights.setFromDate(r_hdo.getFromDate());
			rights.setHolder(r_hdo.getHolder());
			rights.setToDate(r_hdo.getToDate());
			rights.setType(r_hdo.getType());
			retval.setRights(rights);
		}
		
		retval.setSection(hdo.getSection());
		retval.setSite(hdo.getSite());
		retval.setSummary(hdo.getSummary());
		retval.setSummaryTitle(hdo.getSummaryTitle());
		retval.setTitle(hdo.getTitle());
		copyInstructions(retval,hdo);
		return retval;
	}
	public static MimsyDataDTO toDataDTO(Session sess,BaseMimsyDataHDO hdo)
	{
		MimsyDataDTO retval=null;
	
		if (hdo.getDataType()==MimsyDataTypeEnum.OBJECT)
		{
			retval = convert((MimsyObjectDataHDO) hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.MEDIA)
		{
			retval = convert((MimsyMediaDataHDO) hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.GROUP)
		{
			retval = convert(sess,(MimsyCuratedGroupHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.AGENT)
		{
			retval = convert((MimsyAgentDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.EVENT)
		{
			retval = convert((MimsyEventDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.PLACE)
		{
			retval = convert((MimsyPlaceDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.PUBLICATION)
		{
			retval = convert((MimsyPublicationDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.SITE)
		{
			retval = convert((MimsySiteDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.SUBJECT)
		{
			retval = convert((MimsySubjectDataHDO)hdo);
		}
		else if (hdo.getDataType()==MimsyDataTypeEnum.DATA_CONTEXT)
		{
			retval = convert(sess,(CIIMDataContextHDO)hdo);
		}
		return retval;		
	}
	
	public BaseMimsyDataHDO getHDO(Session sess,String mimsy_identifier)throws Exception
	{
		return BaseMimsyDataHDO.lookup(sess, mimsy_identifier);
	}
	
	public boolean isTopGroup(Session sess,String group_id)throws DataException
	{
		return(getTopGroupIdentifiers(sess).contains(group_id));
	}
	
	public static GroupHierarchyDTO convert(Session session, GroupHierarchyXRTreeHDO hdo, String top_group_identifier)
	{
		String mimsy_identifier = top_group_identifier+"/hierarchy-"+hdo.getId();
		String tree_index = hdo.getTree_id();
		String group_identifier=hdo.getCuratedGroup().getMimsyIdentifier();
		GroupHierarchyDTO dto = new GroupHierarchyDTO(mimsy_identifier, tree_index, top_group_identifier,group_identifier, new Date());
		dto.setIndex(hdo.getPosition().toString());
		if(hdo.getCurrent()==false)
			dto.setDataStatus(DataStatusEnum.ARCHIVED);
		return dto;
	}
	
	@SuppressWarnings("unchecked")
	public static List<DataContextDTO> getDataContexts(Session sess,String group_identifier)throws DataException
	{
		List<DataContextDTO> retval = new ArrayList<DataContextDTO>();
		
		Query q = sess.createQuery("SELECT x from com.k_int.aggr2.mimsy.data.hdo.CIIMDataContextHDO x where x.group.mimsyIdentifier=?");
		q.setString(0, group_identifier);
		Iterator<CIIMDataContextHDO> i  = q.iterate();
		while(i.hasNext())
		{
			CIIMDataContextHDO hdo 				= i.next();
			CIIMDataContextMetadataHDO metadata = hdo.getCIIMMetadata();
			MimsyCuratedGroupHDO group 			= hdo.getGroup();
			BaseMimsyDataHDO data_hdo 			= hdo.getMaster();
			DataContextDTO dto 					= new DataContextDTO(hdo.getMimsyIdentifier(), data_hdo.getMimsyIdentifier(),group.getMimsyIdentifier(), hdo.getLastUpdated(), hdo.getGroup().getMemberType(),hdo.getOriginator());
			dto.setConformance(ObjectConformanceEnum.CORE);
			if(metadata!=null)
			{
				CIIMContextDataDTO context = new CIIMContextDataDTO();
				if(metadata.getBoost()!=null)
					context.setBoostValue(metadata.getBoost());
				if(metadata.getFeatured()!=null)
					context.setFeatured(metadata.getFeatured());
				
				 /* extract extension schema data */
				 Iterator<SchemaElementHDO> iter = metadata.getOptions().keySet().iterator();
				 
				 while(iter.hasNext())
				 {
					 SchemaElementHDO key = iter.next();
					 SchemaOptionValueHDO schema_opt_value_hdo = metadata.getOptions().get(key);
					 
					 if(schema_opt_value_hdo != null && key.getDefinition() != null)
					 {
						 ExtensionFieldDTO extension_field = new ExtensionFieldDTO(key.getDefinition().getName(), key.getDefinition().getSolrIdentifier(), schema_opt_value_hdo.getValues());
						 context.getExtensionFields().add(extension_field);
					 }	 
				 }
				
				dto.setCIIMData(context);
			}
			MimsyDataDTO master_dto = toDataDTO(sess,data_hdo);
			CIIMMasterDataDTO copy_data = new CIIMMasterDataDTO(master_dto);					
			dto.setMasterData(copy_data);
			retval.add(dto);			
		}
		return retval;		
	}
	
	
	@SuppressWarnings("unchecked")
	public static List<GroupHierarchyDTO> getHierarchyStructure(Session sess, String top_group_identifier)throws DataException
	{
		List<GroupHierarchyDTO> retval = new ArrayList<GroupHierarchyDTO>();
		
		if(getTopGroupIdentifiers(sess).contains(top_group_identifier))// && hierarchyNeedsPublication(sess,top_group_identifier))
		{
			 Query q = sess.createQuery("SELECT x.id from com.k_int.aggr2.mimsy.data.hdo.MimsyCuratedGroupHDO x WHERE x.mimsyIdentifier=?");
			 q.setString(0, top_group_identifier);
			 Long id = (Long) q.uniqueResult();			
			 q = sess.createQuery("SELECT t from com.k_int.aggr2.mimsy.data.hdo.CIIMGroupHDO x,com.k_int.aggr2.mimsy.data.hdo.GroupHierarchyXRTreeHDO t where x.curatedGroup.id=t.curatedGroup.id AND t.tree_id LIKE ?");
			 q.setString(0, id+".%");
			 Iterator<GroupHierarchyXRTreeHDO> i = q.iterate();
			 while(i.hasNext())
			 {
				 GroupHierarchyXRTreeHDO val = i.next();			
				 GroupHierarchyDTO dto = convert(sess,val,top_group_identifier);				
				 retval.add(dto) ;
			 }			 
		}		
		return retval;
	}
	
	public static void updateGroupPublicationStatus(Session sess, String group_identifier,GroupPublicationStatusEnum pub_status)
	{
		log.error("Set published called");		
		CIIMGroupHDO hdo = CIIMGroupHDO.lookupGroup(sess, group_identifier);
		hdo.setPublicationStatus(pub_status);
		sess.update(hdo);
		sess.flush();
	}
	
	@SuppressWarnings("unchecked")
	public static void archiveDataContexts(Session sess,BaseMimsyDataHDO hdo) throws DataException
	{
		log.error("Archive contexts called");	
		Query q = sess.createQuery("SELECT x from com.k_int.aggr2.mimsy.data.hdo.CIIMDataContextHDO x WHERE x.dataIdentifier=?");
		q.setString(0,hdo.getMimsyIdentifier());
		Iterator<CIIMDataContextHDO> i = q.iterate();
		while(i.hasNext())
		{
			CIIMDataContextHDO context = i.next();
			context.setDataStatus(DataStatusEnum.ARCHIVED);
			sess.update(context);
			sess.flush();
		}
	}
	
	public static void updateAllGroupStatuses(Session sess, String group_identifier,GroupPublicationStatusEnum pub_status, GroupEditorialStatusEnum ed_status,TopGroupEditorialStatusEnum top_status)
	{
		log.error("Set published called");		
		CIIMGroupHDO hdo = CIIMGroupHDO.lookupGroup(sess, group_identifier);
		hdo.setPublicationStatus(pub_status);
		hdo.setEditorialStatus(ed_status);
		hdo.setTopGroupEditorialStatus(top_status);
		sess.update(hdo);
		sess.flush();
	}
	 
	 @SuppressWarnings("unchecked")
	/**
	 * Always returns the top group as first in the list
	 */
	public static Map<String,MimsyCuratedGroupDTO> getGroupStructure(Session sess, String top_group_identifier, boolean update_status)throws DataException
	 {
		 Map<String,MimsyCuratedGroupDTO> retval = new HashMap<String,MimsyCuratedGroupDTO>();
		 if(getTopGroupIdentifiers(sess).contains(top_group_identifier))
		 {
			 Query q = sess.createQuery("SELECT x.id from com.k_int.aggr2.mimsy.data.hdo.MimsyCuratedGroupHDO x WHERE x.mimsyIdentifier=?");
			 q.setString(0, top_group_identifier);
			 Long id = (Long) q.uniqueResult();
			
			 q = sess.createQuery("SELECT x from com.k_int.aggr2.mimsy.data.hdo.CIIMGroupHDO x,com.k_int.aggr2.mimsy.data.hdo.GroupHierarchyXRTreeHDO t where x.curatedGroup.id=t.curatedGroup.id AND t.tree_id LIKE ?");
			 q.setString(0, id+".%");
			 
			 Iterator<CIIMGroupHDO> i 	= q.iterate();
			 //MimsyCuratedGroupDTO top	= null;
			 while(i.hasNext())
			 {
				 CIIMGroupHDO val 			= i.next();				
				 MimsyCuratedGroupDTO dto 	= convert(sess,val.getCuratedGroup());
				 CIIMGroupDataDTO data_dto 	= new CIIMGroupDataDTO();
				 dto.setCIIMGroupData(data_dto);
				 data_dto.setEditorialStatus(val.getEditorialStatus());
				 
				 if(val.getPublicationStatus()!=GroupPublicationStatusEnum.QUEUED)				 
					 throw new DataException("group has a status which is not queued");
				 else if(dto.getMimsyIdentifier().equals(top_group_identifier))
				 {				
					 if(val.getPublicationStatus()!=GroupPublicationStatusEnum.QUEUED)// only publish if queued
					 {
						throw new DataException("Top Group not queued for publication");
					 }					 								
					 data_dto.setTop(true);
					 data_dto.setTopGroupEditorialStatus(val.getTopGroupEditorialStatus());
					// retval.add(0, dto);
				 }
				 //else
				//	 retval.add(dto);
				 retval.put(dto.getMimsyIdentifier(),dto);
				
				 								 	
				 if(val.getGroupContext()!=null)
				 {
					 /* extract extension schema data */
					 Iterator<SchemaElementHDO> iter = val.getGroupContext().getOptions().keySet().iterator();
					 
					 while(iter.hasNext())
					 {
						 SchemaElementHDO key = iter.next();
						 SchemaOptionValueHDO schema_opt_value_hdo = val.getGroupContext().getOptions().get(key);
						 
						 if(schema_opt_value_hdo != null && key.getDefinition() != null)
						 {
							 ExtensionFieldDTO extension_field = new ExtensionFieldDTO(key.getDefinition().getName(), key.getDefinition().getSolrIdentifier(), schema_opt_value_hdo.getValues());
							 data_dto.getExtensionFields().add(extension_field);
						 }	 
					 }
					 
					 /* extract media */
					 Iterator<CIIMMediaDescriptionHDO> media = val.getGroupContext().getMedia().iterator();
					 while(media.hasNext())
					 {
						 CIIMMediaDescriptionHDO media_h 		= media.next();
						 MimsyMediaDataHDO media_hdo 			= media_h.getMediaData();
						 if(media_hdo!=null)
						 {
							 MimsyMediaDataDTO media_dto 		= convert(media_hdo);
							 data_dto.getLinkedMedia().add(media_dto);
						 }					 
					 }
					 data_dto.setLongDescription(val.getGroupContext().getLongDescription());	
				 }
			 }
		 }
		 else
			 throw new DataException("Group is not a top group - only top groups can be published");
		 
		 return retval;
	 }
	
	 
	 
	 @SuppressWarnings("unchecked")
	private static List<String> getTopGroupIdentifiers(Session sess) throws DataException
	 {
		  List<String> 	lReturn 	= new ArrayList<String>();	  
		
		  try 
		  {	
			  Query q = sess.createQuery("SELECT x.curatedGroup.mimsyIdentifier from com.k_int.aggr2.mimsy.data.hdo.GroupHierarchyXRTreeHDO x where x.tree_id = CONCAT('', CAST(x.curatedGroup.id as string), '.')");
	    		  		  
	    	  Iterator<String> i = q.iterate();
	    	  while(i.hasNext())
	    	  {
	    		  lReturn.add(i.next());
	    	  }
	    	 
		  } 
		  catch (HibernateException he ) 
		  {
		      throw new DataException(he);
		  } 
		  catch (Exception e ) 
		  {
			  throw new DataException(e);
		  } 
		  return lReturn;
	  }
	 
	 
	 /*private static boolean hierarchyNeedsPublication(Session sess, String top_group_identifier) throws DataException
	 {
		 boolean retval=true;
		  try 
		  {	
			  // will return null if not ready
			  Query  q 	= sess.createQuery("SELECT x.topGroupEditorialStatus from com.k_int.aggr2.mimsy.data.hdo.CIIMGroupHDO x where x.curatedGroup.mimsyIdentifier=? and x.topGroupEditorialStatus!=0");	  
	    	  q.setString(0, top_group_identifier);
			  Object o 	= q.uniqueResult();
			  if(o==null)
				  retval=false;
		  } 
		  catch (HibernateException he ) 
		  {
		      throw new DataException(he);
		  } 
		  catch (Exception e ) 
		  {
			  throw new DataException(e);
		  } 
		  return retval;
	  }*/
	
	/*public static List<MimsyCuratedGroupDTO> getGroupHierarchyFromTopGroup(Session sess,String top_group_identifier)throws DataException
	{	
		List<String> ids = getChildGroupIds(top_group_identifier);
		CIIMGroupHDO hdo = CIIMGroupHDO.lookupGroup(sess,group_identifier);
		MimsyCuratedGroupDTO retval=null;
		if(hdo!=null)
		{
			retval= toDataDTO(sess,(CIIMGroupHDO)hdo);
		}
		return retval;
	}
	
	 private static List<String> getTopGroupDbIds(Session sess) throws DataException
	 {
		 return getGidsByTreeIdCriteria(sess,"");
	 }
	  
	 private static List<String> getChildGroupDbIds(Session sess,String tree_id) throws DataException
	 {
		 return getGidsByTreeIdCriteria(sess,tree_id);
	 }
	 
	 public static boolean isTopGroup(Session sess,String identifier)throws DataException
	 {
		 List<String> top_groups = getTopGroupIds(sess);
		 return top_groups.contains(identifier);
	 }
	 
	 private static List<String>  getTopGroupIdentifiers()
	 {
		 
	 }*/
	
	/* private static List<String> getGroupDbIdsByTreeIdCriteria(Session sess,String search_string) throws DataException
	  {
		  List<String> 	lReturn 	= null;	  
		  Session 		lSession 	= null;
		  
		  try 
		  {	
			  Query q = lSession.createQuery("SELECT x.curatedGroup.id from com.k_int.aggr2.mimsy.data.hdo.GroupHierarchyXRTreeHDO x where x.tree_id = CONCAT(?, CAST(x.curatedGroup.id as string), '.')");
	    	  q.setParameter(0, search_string);
	    	      	  
	    	  List<Long> lResults = (List<Long>) q.list();
	    	  
	    	  for(Long lId : lResults)
	    	  {
	    		  if(lReturn == null)
	    		  {
	    			  lReturn = new ArrayList<String>(); 
	    		  }
	    		  lReturn.add(lId.toString()); 
	    	  }
		 
			  lSession.clear();
		  } 
		  catch (HibernateException he ) 
		  {
		      throw new DataException(he);
		  } 
		  catch (Exception e ) 
		  {
			  throw new DataException(e);
		  } 
		  return lReturn;
	  }
	*/
	 // status was not set so was change on 22.3.11
	private static MimsyCuratedGroupDTO convert(Session sess,MimsyCuratedGroupHDO hdo)
	{	
		MimsyCuratedGroupDTO retval 	= new MimsyCuratedGroupDTO(hdo.getMimsyIdentifier(), hdo.getLastUpdated(), hdo.getOriginator());
		
		retval.setConformance(hdo.getConformance());
		retval.setDepartment(hdo.getDepartment());
		retval.setDescription(hdo.getDescription());
		retval.setMemberType(hdo.getMemberType());
		retval.setName(hdo.getName());
		retval.setMimsyOwner(hdo.getOwner());
		retval.setPurpose(hdo.getPurpose());
		retval.setDataStatus(hdo.getDataStatus());
		copyInstructions(retval,hdo);
		
		return retval;	
	}
	
	public static BaseMimsyDataHDO toDataHDO(Session sess,MimsyDataDTO dto, Long checksum)throws DataException
	{
		BaseMimsyDataHDO retval;
		//log.error("Attempting to convert "+dto.getClass());
		//log.error("Type is "+dto.getDataType());
		try
		{
			if(dto.getDataType()==MimsyDataTypeEnum.AGENT)
			{			
				MimsyAgentDataHDO hdo = MimsyAgentDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updateAgentHDO(sess,hdo,(MimsyAgentDataDTO)dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.GROUP)
			{
				MimsyCuratedGroupHDO hdo = MimsyCuratedGroupHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier(),((MimsyCuratedGroupDTO)dto).getOriginator());
				updateCuratedGroupHDO(sess,hdo,(MimsyCuratedGroupDTO) dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.DATA_CONTEXT)
			{
				DataContextDTO context 		= (DataContextDTO) dto;			
				String data_identifier		= context.getMimsyDataIdentifier();
				String group_identifier		= context.getGroupIdentifier();
				MimsyDataTypeEnum type      = context.getMasterDataType();
				
				BaseMimsyDataHDO data 		= lookupOrCreateMasterData(sess,data_identifier,type);
				MimsyCuratedGroupHDO group 	= MimsyCuratedGroupHDO.lookupDataOrCreateStub(sess, group_identifier,OriginatorEnum.UNKNOWN);
				CIIMDataContextHDO hdo 		= CIIMDataContextHDO.lookupDataOrCreateStub(sess,dto.getMimsyIdentifier(),group,(MimsyObjectDataHDO)data,context.getOriginator());				
				updateDataContextHDO(sess,hdo,(DataContextDTO) dto,checksum );
				retval=hdo;			
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.EVENT)
			{	
				MimsyEventDataHDO hdo = MimsyEventDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updateEventHDO(sess,hdo,(MimsyEventDataDTO) dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.MEDIA)
			{
				MimsyMediaDataHDO hdo = MimsyMediaDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier(),((MimsyMediaDataDTO)dto).getOriginator());
				updateMediaHDO(sess,hdo,(MimsyMediaDataDTO)dto,checksum);
				retval=hdo;
			}	
			else if (dto.getDataType()==MimsyDataTypeEnum.OBJECT)
			{
				MimsyObjectDataHDO hdo = MimsyObjectDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updateObjectHDO(sess,hdo,(MimsyObjectDataDTO)dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.PLACE)
			{				
				MimsyPlaceDataHDO hdo = MimsyPlaceDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updatePlaceHDO(hdo,(MimsyPlaceDataDTO)dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.PUBLICATION)
			{
				MimsyPublicationDataHDO hdo = MimsyPublicationDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updatePublicationHDO(sess,hdo,(MimsyPublicationDataDTO)dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.SITE)
			{
				MimsySiteDataHDO hdo = MimsySiteDataHDO.lookupDataOrCreateStub(sess,dto.getMimsyIdentifier());
				updateSiteHDO(hdo,(MimsySiteDataDTO)dto,checksum);
				retval=hdo;
			}
			else if (dto.getDataType()==MimsyDataTypeEnum.SUBJECT)
			{
				MimsySubjectDataHDO hdo = MimsySubjectDataHDO.lookupDataOrCreateStub(sess, dto.getMimsyIdentifier());
				updateSubjectHDO(hdo,(MimsySubjectDataDTO) dto,checksum);
				retval=hdo;
			}
			else
				throw new DataException("No converter for dto "+dto.getClass());
			
			
			retval.setChecksum(checksum);
			return retval;
		}
		catch(Exception e)
		{
			throw new DataException(e);
		}
	}
	
	private static void updateAgentHDO(Session sess,MimsyAgentDataHDO hdo, MimsyAgentDataDTO dto, long checksum)
	{		
		hdo.setBirthDate(dto.getBirthDate());
		hdo.setBirthPlace(dto.getBirthPlace());
		hdo.setBriefBio(dto.getBriefBio());
		hdo.setCauseOfDeath(dto.getCauseOfDeath());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDeathDate(dto.getDeathDate());
		hdo.setDeathPlace(dto.getDeathPlace());
		hdo.setDeceased(dto.isDeceased());
		hdo.setDescription(dto.getDescription());
		hdo.setFirstName(dto.getFirstName());
		hdo.setGender(dto.getGender());
		hdo.setHonorarySuffix(dto.getHonorarySuffix());
		hdo.setIndividual(dto.isIndividual());
		copyInstructions(hdo,dto);
		hdo.setLastName(dto.getLastName());
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setNationality(dto.getNationality());
		hdo.setOccupation(dto.getOccupation());
		hdo.setPreferredName(dto.getPreferredName());
		hdo.setSuffix(dto.getSuffix());
		hdo.setTitlePrefix(dto.getTitlePrefix());
		hdo.getVariations().clear();
		hdo.setVariations(copyTextTypePairs(dto.getVariations()));
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updateCuratedGroupHDO(Session sess, MimsyCuratedGroupHDO hdo, MimsyCuratedGroupDTO dto, long checksum)
	{
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDepartment(dto.getDepartment());
		hdo.setDataStatus(dto.getStatus());
		copyInstructions(hdo,dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setMemberType(dto.getMemberType());
		hdo.setName(dto.getName());
		hdo.setDescription(dto.getDescription());
		hdo.setPurpose(dto.getPurpose());
		hdo.setOwner(dto.getMimsyOwner());
		if(hdo.getOriginator()==OriginatorEnum.UNKNOWN)
			hdo.setOriginator(dto.getOriginator());
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updateDataContextHDO(Session sess, CIIMDataContextHDO hdo, DataContextDTO dto, long checksum)
	{			
		hdo.setDataStatus(dto.getStatus());
		hdo.setChecksum(checksum);
		hdo.setDataStatus(dto.getStatus());
		copyInstructions(hdo, dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updateEventHDO(Session sess,MimsyEventDataHDO hdo, MimsyEventDataDTO dto, long checksum)
	{
		hdo.setBriefName(dto.getBriefName());
		hdo.setCandidate(dto.getCandidate());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDescription(dto.getDescription());
		hdo.setDisplayDate(dto.getDisplayDate());
		hdo.setDuration(dto.getDuration());
		hdo.setEndDate(dto.getEndDate());
		hdo.setFullName(dto.getFullName());
		copyInstructions(hdo, dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setScopeNote(dto.getScopeNote());
		hdo.setStartDate(dto.getStartDate());
		hdo.setUnits(dto.getUnits());
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updateMediaHDO(Session sess,MimsyMediaDataHDO hdo, MimsyMediaDataDTO dto, long checksum)
	{
		hdo.setCabinetImageNumber(dto.getCabinetImageNumber());
		hdo.setCaption(dto.getCaption());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setCreditLine(dto.getCreditLine());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDonorFilename(dto.getDonorFilename());
		hdo.setDonorLocation(dto.getDonorLocation());
		hdo.setDonorRelativePath(dto.getDonorRelativePath());
		hdo.setFormat(dto.getFormat());
		copyInstructions(hdo, dto);
		hdo.setLargeDonorLocation(dto.getLargeDonorLocation());
		hdo.setLargeDonorRelativePath(dto.getLargeDonorRelativePath());
		hdo.setLargeFileLocation(dto.getLargeFileLocation());
		hdo.setLargeImageType(dto.getLargeImageType());
		hdo.setLargeRelativePath(dto.getLargeRelativePath());
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setMediaType(dto.getMediaType());
		hdo.setMedium(dto.getMedium());
		hdo.setMidFileLocation(dto.getMidFileLocation());
		hdo.setMidRelativePath(dto.getMidRelativePath());
		hdo.setPreviewFileLocation(dto.getPreviewFileLocation());
		hdo.setPreviewRelativePath(dto.getPreviewRelativePath());
		hdo.setRecordType(dto.getRecordType());
		hdo.setRelationship(dto.getRelationship());
		copyMediaRights(hdo,dto);
		hdo.setSummaryTitle(dto.getSummaryTitle());
		
	}
	
	private static void updateObjectHDO(Session sess,MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto, long checksum)throws Exception
	{
		hdo.setAcquisition(dto.getAcquisition());
		hdo.setAdditionalInformation(dto.getAdditionalInformation());
		hdo.getBoroughs().clear();
		Iterator<String> i = dto.getBoroughs().iterator();
		while(i.hasNext())
		{
			String s = i.next();
			if(s==null)
				continue;
			StringReplacementHDO rep = new StringReplacementHDO(s);
			hdo.getBoroughs().add(rep);
		}
		
		hdo.setChecksum(checksum);
		hdo.getCollectedDates().clear();
		Iterator<MimsyDateDTO> dates = dto.getCollectedDates().iterator();
		while(dates.hasNext())
		{
			MimsyDateDTO date = dates.next();
			if(date.getDateText()==null)
				continue;
			MimsyDateHDO date_h = new MimsyDateHDO(date.getDateText(), date.getPrior());
			if(date.getAttribution()!=null)
			{
				AttributionDTO attrib = date.getAttribution();
				AttributionHDO a_h = new AttributionHDO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
				date_h.setAttribution(a_h);
			}
			date_h.setEarliest(date.getEarliest());
			date_h.setEra(date.getEra());
			date_h.setLatest(date.getLatest());
			date_h.setMethod(date.getMethod());
			date_h.setRelation(date.getRelation());
			hdo.getCollectedDates().add(date_h);
		}
		//hdo.getCollectorNames().clear();
		//i = dto.getCollectorNames().iterator();
		//while(i.hasNext())
		//{
		//	String s = i.next();
		//	if(s==null)
		//		continue;
			//StringReplacementHDO rep = new StringReplacementHDO(s);
			
		//	hdo.getCollectorNames().add(rep);
		//}
		
		
		hdo.setConformance(dto.getConformance());
		hdo.setConservationAction(dto.getConservationAction());
		hdo.setCreditLine(dto.getCreditLine());
		if(dto.getCurrentLocation()!=null)
		{
			LocationHDO location = LocationHDO.lookupOrCreateByCode(sess,dto.getCurrentLocation());
			location.setName(dto.getCurrentLocationDescription());
			hdo.setCurrentLocation(location);
		}
		
		hdo.setDataStatus(dto.getStatus());
		hdo.getDescriptions().clear();
		Iterator<DescriptionDTO> descs = dto.getDescriptions().iterator();
		while(descs.hasNext())
		{
			DescriptionDTO desc = descs.next();
			DescriptionHDO d_hdo = new DescriptionHDO(desc.getDescription(), desc.getType(), desc.getPrior());
			d_hdo.setDate(desc.getDate());
			d_hdo.setSource(desc.getSource());
			hdo.getDescriptions().add(d_hdo);
			
		}
		
		hdo.setDisplayDate(dto.getDisplayDate());
		hdo.getFunctionalClassification().clear();
		Iterator<NameDTO> names = dto.getFunctionalClassification().iterator();
		while(names.hasNext())
		{
			NameDTO name = names.next();
			NameHDO name_h = new NameHDO(name.getName(), name.getType(), name.getPrevious());
		
			hdo.getFunctionalClassification().add(name_h);
		}
		
		copyInstructions(hdo, dto);
		hdo.setIntendedUse(dto.getIntendedUse());
		hdo.setItemCount(dto.getItemCount());
		hdo.setLastUpdated(dto.getLastUpdated());
		updateLinkedAgents(sess,hdo,dto);
		updateLinkedCollectors(sess,hdo,dto);
		updateLinkedEvents(sess,hdo,dto);
		updateLinkedMakers(sess,hdo,dto);
		updateLinkedMedia(sess,hdo,dto);
		updateLinkedObjects(sess,hdo,dto);
		updateLinkedPlaces(sess,hdo,dto);
		updateLinkedPublications(sess,hdo,dto);
		updateLinkedSites(sess,hdo,dto);
		updateLinkedSubjects(sess,hdo,dto);
		hdo.setLoanAllowed(dto.isLoanAllowed());
		dates = dto.getMadeDates().iterator();
		hdo.getMadeDates().clear();
		while(dates.hasNext())
		{
			MimsyDateDTO date = dates.next();
			if(date.getDateText()==null)
				continue;
			MimsyDateHDO date_h = new MimsyDateHDO(date.getDateText(), date.getPrior());
			if(date.getAttribution()!=null)
			{
				AttributionDTO attrib = date.getAttribution();
				AttributionHDO a_h = new AttributionHDO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
				date_h.setAttribution(a_h);
			}
			date_h.setEarliest(date.getEarliest());
			date_h.setEra(date.getEra());
			date_h.setLatest(date.getLatest());
			date_h.setMethod(date.getMethod());
			date_h.setRelation(date.getRelation());
			hdo.getMadeDates().add(date_h);
		}
		//i = dto.getMakerNames().iterator();
		//hdo.getMakerNames().clear();
		//while(i.hasNext())
		//{
		//	String s = i.next();
		//	if(s==null)
		//		continue;
		//	StringReplacementHDO rep = new StringReplacementHDO(s);
		//	hdo.getMakerNames().add(rep);
		//}
		//TODO hdo.setMasterContext(context);
		i = dto.getMaterials().iterator();
		hdo.getMaterials().clear();
		while(i.hasNext())
		{
			String s = i.next();
			if(s==null)
				continue;
			StringReplacementHDO rep = new StringReplacementHDO(s);
			hdo.getMaterials().add(rep);
		}
		hdo.setMeasurements(dto.getMeasurements());
		
		names = dto.getNames().iterator();
		hdo.getNames().clear();
		while(names.hasNext())
		{
			NameDTO name = names.next();
			NameHDO name_h = new NameHDO(name.getName(), name.getType(), name.getPrevious());
		
			hdo.getNames().add(name_h);
		}
		
		hdo.setNote(dto.getNote());
		hdo.setObjectId(dto.getObjectId());
		hdo.getOtherNumbers().clear();
		Iterator<TextTypePairDTO> pairs = dto.getOtherNumbers().iterator();
		while(pairs.hasNext())
		{
			TextTypePairDTO pair = pairs.next();
			TextTypePairHDO h = new TextTypePairHDO(pair.getText(), pair.getType());
			hdo.getOtherNumbers().add(h);
		}
		
		Iterator<TitleDTO> titles = dto.getOtherTitles().iterator();
		hdo.getOtherTitles().clear();
		while(titles.hasNext())
		{
			TitleDTO title = titles.next();
			AttributionHDO a_h=null;
			if(title.getAttribution()!=null)
			{
				AttributionDTO attrib = title.getAttribution();
				a_h = new AttributionHDO(attrib.getAttributor(), attrib.getDate());
				a_h.setCertainty(attrib.getCertainty());
				a_h.setComment(attrib.getComment());
				a_h.setSource(attrib.getSource());
				a_h.setType(attrib.getType());				
			}
			TitleHDO h = new TitleHDO(title.getTitle(), title.getType(), title.getPrior(), a_h);
			hdo.getOtherTitles().add(h);
		}
		
		hdo.setOwnerStatus(dto.getOwnerStatus());
		hdo.setPagination(dto.getPagination());
		hdo.setPermanentLocation(dto.getPermanentLocation());
		hdo.setPlaceCollected(dto.getPlaceCollected());
		hdo.setPlaceMade(dto.getPlaceMade());
		if(dto.getPrimaryMaterial()!=null)
		{
			StringReplacementHDO primary_material = new StringReplacementHDO(dto.getPrimaryMaterial());
			sess.save(primary_material);
			hdo.setPrimaryMaterial(primary_material);
		}
		i = dto.getProjectCodes().iterator();
		hdo.getProjectCodes().clear();
		while(i.hasNext())
		{
			String s = i.next();
			if(s==null)
				continue;
			StringReplacementHDO rep = new StringReplacementHDO(s);
			hdo.getProjectCodes().add(rep);
		}
		
		hdo.setProjectNotes(dto.getProjectNotes());
		hdo.setPublisher(dto.getPublisher());
		hdo.setRecordType(dto.getRecordType());
		RightsDTO rights = dto.getRights();
		if(rights!=null)
		{
			RightsHDO h = new RightsHDO();
			h.setCreditLine(rights.getCreditLine());
			h.setDetails(rights.getDetails());
			h.setFromDate(rights.getFromDate());
			h.setHolder(rights.getHolder());
			h.setToDate(rights.getToDate());
			h.setType(rights.getType());			
			hdo.setRights(h);
		}
		hdo.setSection(dto.getSection());
		hdo.setSite(dto.getSite());
		hdo.setSummary(dto.getSummary());
		hdo.setSummaryTitle(dto.getSummaryTitle());
		hdo.setTitle(dto.getTitle());
	}
	
	private static void updateLinkedMedia(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedMedia().clear();
		Iterator<Map.Entry<Integer, MimsyMediaDataDTO>> i = dto.getLinkedMedia().entrySet().iterator();
		while(i.hasNext())
		{
			//TODO check this is okay
			Map.Entry<Integer, MimsyMediaDataDTO> entry = i.next();
			MimsyMediaDataDTO media_dto = entry.getValue();
			Integer position 			= entry.getKey();
			MimsyMediaDataHDO media_hdo = MimsyMediaDataHDO.lookupDataOrCreateStub(sess, media_dto.getMimsyIdentifier(),OriginatorEnum.MIMSY);
			updateMediaHDO(sess, media_hdo, media_dto, MimsyHelper.getChecksum(media_dto));
			MediaReferenceHDO ref 		= new MediaReferenceHDO(media_hdo, media_dto.getRelationship());
			ref.setOwner(hdo);
			ref.setPosition(position);
			sess.save(ref);
			sess.flush();
			hdo.getLinkedMedia().add(ref);			
		}
	}
	
	private static void updateLinkedAgents(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedAgents().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedAgents().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyAgentDataHDO ref_hdo = MimsyAgentDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			AgentReferenceHDO r_hdo = new AgentReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedAgents().add(r_hdo);			
		}
	}
	
	private static void updateLinkedCollectors(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedCollectors().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedCollectors().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyAgentDataHDO ref_hdo = MimsyAgentDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			CollectorReferenceHDO r_hdo = new CollectorReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedCollectors().add(r_hdo);			
		}
	}
	
	private static void updateLinkedEvents(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedEvents().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedEvents().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyEventDataHDO ref_hdo = MimsyEventDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			EventReferenceHDO r_hdo = new EventReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedEvents().add(r_hdo);			
		}
	}
	
	private static void updateLinkedMakers(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedMakers().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedMakers().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyAgentDataHDO ref_hdo = MimsyAgentDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			MakerReferenceHDO r_hdo = new MakerReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedMakers().add(r_hdo);			
		}
	}
	
	private static void updateLinkedObjects(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedObjects().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedObjects().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyObjectDataHDO ref_hdo = MimsyObjectDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			ObjectReferenceHDO r_hdo = new ObjectReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedObjects().add(r_hdo);			
		}
	}
	
	private static void updateLinkedPlaces(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedPlaces().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedPlaces().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyPlaceDataHDO ref_hdo = MimsyPlaceDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			PlaceReferenceHDO r_hdo = new PlaceReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedPlaces().add(r_hdo);			
		}
	}
	
	private static void updateLinkedPublications(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedPublications().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedPublications().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsyPublicationDataHDO ref_hdo = MimsyPublicationDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			PublicationReferenceHDO r_hdo = new PublicationReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedPublications().add(r_hdo);			
		}
	}
	
	private static void updateLinkedSites(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedSites().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedSites().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsySiteDataHDO ref_hdo = MimsySiteDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			SiteReferenceHDO r_hdo = new SiteReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedSites().add(r_hdo);			
		}
	}
	
	private static void updateLinkedSubjects(Session sess, MimsyObjectDataHDO hdo, MimsyObjectDataDTO dto)throws Exception
	{
		hdo.getLinkedSubjects().clear();
		Iterator<ControlledReferenceDTO> i = dto.getLinkedSubjects().iterator();
		while(i.hasNext())
		{
			ControlledReferenceDTO ref = i.next();
			
			MimsySubjectDataHDO ref_hdo = MimsySubjectDataHDO.lookupDataOrCreateStub(sess,ref.getIdentifier());
			
			SubjectReferenceHDO r_hdo = new SubjectReferenceHDO(ref_hdo, ref.getRelationship());
			r_hdo.setOwner(hdo);
			
			sess.save(r_hdo);
			sess.flush();
			hdo.getLinkedSubjects().add(r_hdo);			
		}
	}
	
	
	
	private static void updatePlaceHDO(MimsyPlaceDataHDO hdo,MimsyPlaceDataDTO dto,long checksum)
	{
		hdo.setBroaderPlace(dto.getBroaderPlace());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDescription(dto.getDescription());
		copyInstructions(hdo, dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setLevelNumber(dto.getLevelNumber());
		hdo.setNote(dto.getNote());
		hdo.setPlace(dto.getPlace());
		hdo.setPlaceType(dto.getPlaceType());
		copyPlaceVariations(hdo,dto);
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updatePublicationHDO(Session sess,MimsyPublicationDataHDO hdo, MimsyPublicationDataDTO dto, long checksum)throws Exception
	{
		hdo.setAuthor(dto.getAuthor());
		hdo.setBriefTitle(dto.getBriefTitle());
		hdo.setChecksum(checksum);
		hdo.setCitation(dto.getCitation());
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDescription(dto.getDescription());
		hdo.setEdition(dto.getEdition());
		hdo.setEditor(dto.getEditor());
		hdo.setIllustrated(dto.getIllustrated());
		copyInstructions(hdo,dto);
		hdo.setIsbnIssn(dto.getIsbnIssn());
		hdo.setIssue(dto.getIssue());
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setLocation(dto.getLocation());
		hdo.setPagesTracks(dto.getPagesTracks());
		hdo.setPlacePublished(dto.getPlacePublished());
		hdo.setPublicationDate(dto.getPublicationDate());
		hdo.setPublicationType(dto.getPublicationType());
		hdo.setPublished(dto.isPublished());
		hdo.setPublisher(dto.getPublisher());
		hdo.setSeriesTitle(dto.getSeriesTitle());
		copySubjectHeadings(sess,hdo,dto);
		hdo.setTitle(dto.getTitle());
		hdo.setTitleStatement(dto.getTitleStatement());
		hdo.setVolume(dto.getVolume());
		hdo.setVolumes(dto.getVolumes());
		hdo.setYearPublished(dto.getYearPublished());
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void updateSiteHDO(MimsySiteDataHDO hdo, MimsySiteDataDTO dto, long checksum)
	{
		hdo.setBorough(dto.getBorough());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDate(dto.getDate());
		hdo.setEastings(dto.getEastings());
		copyExcavations(hdo,dto);
		copyInstructions(hdo,dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setLocation(dto.getLocation());
		hdo.setName(dto.getName());
		hdo.setNorthings(dto.getNorthings());
		hdo.setOrganisation(dto.getOrganisation());
		hdo.setSiteType(dto.getSiteType());
		hdo.setSummary(dto.getSummary());
		hdo.setSiteYear(dto.getSiteYear());		
		hdo.setSummaryTitle(dto.getSummaryTitle());
		hdo.setSiteId(dto.getSiteId());
	}
	
	private static void updateSubjectHDO(MimsySubjectDataHDO hdo, MimsySubjectDataDTO dto, long checksum)
	{
		hdo.setArea(dto.getArea());
		hdo.setCategory(dto.getCategory());
		hdo.setChecksum(checksum);
		hdo.setConformance(dto.getConformance());
		hdo.setDataStatus(dto.getStatus());
		hdo.setDescription(dto.getDescription());
		copyInstructions(hdo, dto);
		hdo.setLastUpdated(dto.getLastUpdated());
		hdo.setPeriod(dto.getPeriod());
		hdo.setSubCategory(dto.getSubCategory());
		hdo.setSubject(dto.getSubject());
		hdo.setSummaryTitle(dto.getSummaryTitle());
	}
	
	private static void copyExcavations(MimsySiteDataHDO hdo, MimsySiteDataDTO dto)
	{
		hdo.getExcavations().clear();
		Iterator<ExcavationDTO> i = dto.getExcavations().iterator();
		while(i.hasNext())
		{
			ExcavationDTO var = i.next();
			ExcavationHDO v_hdo = new ExcavationHDO();
			v_hdo.setEnd(var.getEnd());
			v_hdo.setOrganisation(var.getOrganisation());
			v_hdo.setStart(var.getStart());
			v_hdo.setRole(var.getRole());
			hdo.getExcavations().add(v_hdo);
		}
	}
	
	
	private static void copySubjectHeadings(Session sess,MimsyPublicationDataHDO hdo, MimsyPublicationDataDTO dto) throws Exception
	{
		hdo.getSubjectHeadings().clear();
		Iterator<String> i = dto.getSubjectHeadings().iterator();
		while(i.hasNext())
		{
			String subject = i.next();
			SubjectHeadingHDO subject_h = SubjectHeadingHDO.lookupOrCreate(sess,subject);
			hdo.getSubjectHeadings().add(subject_h);			
		}
	}
	
	private static void copyPlaceVariations(MimsyPlaceDataHDO hdo, MimsyPlaceDataDTO dto)
	{
		hdo.getVariations().clear();
		Iterator<PlaceVariationDTO> i = dto.getVariations().iterator();
		while(i.hasNext())
		{
			PlaceVariationDTO var = i.next();
			PlaceVariationHDO v_hdo = new PlaceVariationHDO(var.getText(), var.getType());
			v_hdo.setEnd(var.getEnd());
			v_hdo.setNote(var.getNote());
			v_hdo.setStart(var.getStart());
			hdo.getVariations().add(v_hdo);
		}
	}
	private static void copyMediaRights(MimsyMediaDataHDO hdo, MimsyMediaDataDTO dto)
	{
		hdo.getRights().clear();
		Iterator<RightsDTO> i = dto.getRights().iterator();
		while(i.hasNext())
		{
			RightsDTO rights = i.next();
			RightsHDO r_hdo = new RightsHDO();
			r_hdo.setCreditLine(rights.getCreditLine());
			r_hdo.setDetails(rights.getDetails());
			r_hdo.setFromDate(rights.getFromDate());
			r_hdo.setHolder(rights.getHolder());
			r_hdo.setToDate(rights.getToDate());
			r_hdo.setType(rights.getType());
			hdo.getRights().add(r_hdo);			
		}	
	}
	
	public static BaseMimsyDataHDO updateHDO(Session sess,BaseMimsyDataHDO hdo, MimsyDataDTO dto, long checksum) throws DataException
	{	
		if(dto.getDataType()==MimsyDataTypeEnum.AGENT)		
			updateAgentHDO(sess,(MimsyAgentDataHDO)hdo,(MimsyAgentDataDTO)dto,checksum);	
		else if (dto.getDataType()==MimsyDataTypeEnum.GROUP)
			updateCuratedGroupHDO(sess,(MimsyCuratedGroupHDO)hdo,(MimsyCuratedGroupDTO)dto,checksum);
		else if (dto.getDataType()==MimsyDataTypeEnum.DATA_CONTEXT)
			updateDataContextHDO(sess,(CIIMDataContextHDO)hdo,(DataContextDTO) dto,checksum );
		else if (dto.getDataType()==MimsyDataTypeEnum.EVENT)
			updateEventHDO(sess,(MimsyEventDataHDO)hdo,(MimsyEventDataDTO) dto,checksum);
		else if (dto.getDataType()==MimsyDataTypeEnum.MEDIA)
			updateMediaHDO(sess,(MimsyMediaDataHDO)hdo,(MimsyMediaDataDTO)dto,checksum);
		else if (dto.getDataType()==MimsyDataTypeEnum.OBJECT)
		{
			try
			{
				updateObjectHDO(sess,(MimsyObjectDataHDO)hdo,(MimsyObjectDataDTO)dto,checksum);
			}
			catch(Exception e)
			{
				throw new DataException(e);
			}
		}
		else if (dto.getDataType()==MimsyDataTypeEnum.PLACE)
			updatePlaceHDO((MimsyPlaceDataHDO)hdo,(MimsyPlaceDataDTO)dto,checksum);
		else if (dto.getDataType()==MimsyDataTypeEnum.PUBLICATION)
		{
			try
			{
				updatePublicationHDO(sess,(MimsyPublicationDataHDO)hdo,(MimsyPublicationDataDTO)dto,checksum);		
			}
			catch(Exception e)
			{
				throw new DataException(e);
			}
		}
		else if (dto.getDataType()==MimsyDataTypeEnum.SITE)
			updateSiteHDO((MimsySiteDataHDO)hdo,(MimsySiteDataDTO)dto,checksum);
		else if (dto.getDataType()==MimsyDataTypeEnum.SUBJECT)
			updateSubjectHDO((MimsySubjectDataHDO)hdo,(MimsySubjectDataDTO) dto,checksum);		
		else
			throw new DataException("No converter for dto "+dto.getClass());
	
		return hdo;
	}
	
	private static Set<TextTypePairHDO> copyTextTypePairs(Set<TextTypePairDTO> data)
	{
		Set<TextTypePairHDO> retval = new HashSet<TextTypePairHDO>();
		Iterator<TextTypePairDTO> i = data.iterator();
		while(i.hasNext())
		{
			TextTypePairDTO instr = i.next();
			TextTypePairHDO i_hdo = new TextTypePairHDO(instr.getText(),instr.getType());			
			retval.add(i_hdo);
		}
		return retval;
	}
	
	private static void copyInstructions(BaseMimsyDataHDO hdo, MimsyDataDTO dto)
	{
		hdo.getInstructions().clear();
		Iterator<InstructionDTO> i = dto.getInstructions().iterator();
		while(i.hasNext())
		{
			InstructionDTO instr = i.next();
			InstructionHDO i_hdo = new InstructionHDO();
			i_hdo.setDate(instr.getDate());
			i_hdo.setInstruction(instr.getInstruction());
			i_hdo.setName(instr.getName());
			i_hdo.setStatus(instr.getStatus());
			i_hdo.setType(instr.getType());
			hdo.getInstructions().add(i_hdo);
		}
		
	}
	
	private static BaseMimsyDataHDO lookupOrCreateMasterData(Session sess,String mimsy_identifier, MimsyDataTypeEnum type)throws Exception
	{
		BaseMimsyDataHDO retval=null;
		if(type==MimsyDataTypeEnum.AGENT)
		{
			retval = MimsyAgentDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.EVENT)
		{
			retval = MimsyEventDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.MEDIA)
		{
			retval = MimsyMediaDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier,OriginatorEnum.MIMSY);
		}
		else if(type==MimsyDataTypeEnum.OBJECT)
		{
			retval = MimsyObjectDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.PLACE)
		{
			retval = MimsyPlaceDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.PUBLICATION)
		{
			retval = MimsyPublicationDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.SITE)
		{
			retval = MimsySiteDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else if(type==MimsyDataTypeEnum.SUBJECT)
		{
			retval = MimsySubjectDataHDO.lookupDataOrCreateStub(sess, mimsy_identifier);
		}
		else
			throw new Exception("Unable to create a context with content type "+type);
		
		return retval;
	}
	
}