=== added file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dxf2/service/StaXDataValueImportService.java 2011-09-07 19:26:45 +0000 @@ -0,0 +1,490 @@ +package org.hisp.dhis.importexport.dxf2.service; + +/* + * Copyright (c) 2004-2005, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * + * @author bobj + * @version created 05-Sep-2011 + */ +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import static org.apache.commons.lang.StringUtils.defaultIfEmpty; + +import javax.xml.namespace.QName; + +import org.amplecode.quick.BatchHandler; +import org.amplecode.quick.BatchHandlerFactory; +import org.amplecode.staxwax.reader.XMLReader; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.common.ProcessState; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo; +import org.hisp.dhis.dataelement.DataElementCategoryService; +import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.dataset.DataSetService; +import org.hisp.dhis.datavalue.DataValue; +import org.hisp.dhis.importexport.ImportException; +import org.hisp.dhis.importexport.ImportParams; +import org.hisp.dhis.importexport.dxf2.model.*; +import org.hisp.dhis.jdbc.batchhandler.DataValueBatchHandler; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.user.CurrentUserService; + +/** + * Really basic DXF2 class for reading data + * + * @author bobj + * @version created 5-Sep-2011 + */ +public class StaXDataValueImportService +{ + + private static final Log log = LogFactory.getLog( StaXDataValueImportService.class ); + + // --------------------------------------------------------------------------- + // Status/Log messages + // TODO: internationalise these + // --------------------------------------------------------------------------- + public static final String NO_DATAVALUESETS = "There are no datasets in this message"; + + public static final String IMPORTING_DATAVALUES = "Importing data values"; + + public static final String INVALID_PERIOD = "Invalid period : %s"; + + public static final String NO_SUCH_ORGUNIT = "No such orgunit : %s = %s"; + + public static final String NO_SUCH_DATAELEMENT = "No such dataElement : %s = %s"; + + public static final String NO_ROOT = "Couldn't find dxf root element"; + + public static final String UNKNOWN_ID_STRATEGY = "Unknown id strategy = %s"; + + public static final String SUCCESS = "DataValue import complete"; + + // ------------------------------------------------------------------------- + // Dependencies + // ------------------------------------------------------------------------- + + private CurrentUserService currentUserService; + + public void setCurrentUserService( CurrentUserService currentUserService ) + { + this.currentUserService = currentUserService; + } + + private DataValueSetService dataValueSetService; + + public void setDataValueSetService( DataValueSetService dataValueSetService ) + { + this.dataValueSetService = dataValueSetService; + } + + private DataSetService dataSetService; + + public void setDataSetService( DataSetService dataSetService ) + { + this.dataSetService = dataSetService; + } + + private DataElementService dataElementService; + + public void setDataElementService( DataElementService dataElementService ) + { + this.dataElementService = dataElementService; + } + + private DataElementCategoryService categoryService; + + public void setCategoryService( DataElementCategoryService categoryService ) + { + this.categoryService = categoryService; + } + + private OrganisationUnitService organisationUnitService; + + public void setOrganisationUnitService( OrganisationUnitService organisationUnitService ) + { + this.organisationUnitService = organisationUnitService; + } + + private PeriodService periodService; + + public void setPeriodService( PeriodService periodService ) + { + this.periodService = periodService; + } + + private BatchHandlerFactory batchHandlerFactory; + + public void setBatchHandlerFactory( BatchHandlerFactory batchHandlerFactory ) + { + this.batchHandlerFactory = batchHandlerFactory; + } + + public void read( XMLReader reader, ImportParams params, ProcessState state ) + { + String user = currentUserService.getCurrentUsername(); + + BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( + DataValueBatchHandler.class ).init(); + + int cumulativeDataValueCounter = 0; + + try + { + if ( !reader.moveToStartElement( Dxf.DXFROOT, Dxf.DXFROOT ) ) + { + throw new ImportException( NO_ROOT ); + } + QName rootName = reader.getElementQName(); + + params.setNamespace( defaultIfEmpty( rootName.getNamespaceURI(), Dxf.NAMESPACE_20 ) ); + String version = reader.getAttributeValue( Dxf.ATTRIBUTE_MINOR_VERSION ); + params.setMinorVersion( version != null ? version : Dxf.MINOR_VERSION_10 ); + + log.debug( String.format( "Importing %s minor version %s", rootName.getNamespaceURI(), version ) ); + + // move straight to the DataValue sets + // we are not looking for metadata + reader.moveToStartElement( Dxf.DATAVALUESETS ); + + Date timestamp = new Date(); + + if ( !reader.isStartElement( Dxf.DATAVALUESETS ) ) + { + throw new ImportException( NO_DATAVALUESETS ); + } + + // Outer Loop + // process datavaluesets until no more datavaluesets + + int countDataValueSets = 0; + do + { + // look for a DataValue set + reader.moveToStartElement( Dxf.DATAVALUESET ); + if ( !reader.isStartElement( Dxf.DATAVALUESET ) ) + { + // we have to reach here eventually + break; + } + + // Pick off the attributes + String idSchemeStr = reader.getAttributeValue( DataValueSet.ATTR_IDSCHEME ); + String dataSet = reader.getAttributeValue( DataValueSet.ATTR_DATASET ); + String period = reader.getAttributeValue( DataValueSet.ATTR_PERIOD ); + String outerOrgunit = reader.getAttributeValue( DataValueSet.ATTR_ORGUNIT ); + String comment = reader.getAttributeValue( DataValueSet.ATTR_COMMENT ); + + log.debug( String.format( + "Importing datavalueset (%s): period %s : orgunit %s : idscheme : %s", + comment, period, outerOrgunit, idSchemeStr ) ); + + + // Determine identifier scheme to use + + DataValueSet.IdentificationStrategy idScheme = DataValueSet.DEFAULT_STRATEGY; + + if ( idSchemeStr != null ) + { + try + { + idScheme = idScheme.valueOf( idSchemeStr ); + } catch ( IllegalArgumentException ex ) + { + throw new ImportException( String.format( UNKNOWN_ID_STRATEGY, idSchemeStr ) ); + } + } + + Period outerPeriod = getPeriodObj( period ); + + // maps for translating identifiers + Map dataelementMap = null; + Map orgunitMap = null; + + // get map for translating dataelement identifiers + dataelementMap = getDataElementMap( dataSet, idScheme ); + + Integer outerOrgunitId = null; + // if orgunit defined at datavalueset level, use it + if ( outerOrgunit != null ) + { + outerOrgunitId = getOrgUnitByIdentifier( outerOrgunit, idScheme ).getId(); + } else + { + // get map for translating orgunit identifiers + orgunitMap = getOrgUnitMap( dataSet, idScheme ); + } + + // only supporting default optioncombo at present + DataElementCategoryOptionCombo optioncombo = + categoryService.getDefaultDataElementCategoryOptionCombo(); + + int countDataValues = 0; + // process datavalues - loop until no more datavalues + do + { + // look for a DataValue + reader.moveToStartElement( DataValueSet.DATAVALUE ); + if ( !reader.isStartElement( DataValueSet.DATAVALUE ) ) + { + // we have to reach here eventually + break; + } + log.debug( "Reading Datavalue" ); + + String dataElementId = reader.getAttributeValue( + org.hisp.dhis.importexport.dxf2.model.DataValue.ATTR_DATAELEMENT ); + String innerOrgUnitId = reader.getAttributeValue( + org.hisp.dhis.importexport.dxf2.model.DataValue.ATTR_ORGUNIT ); + String value = reader.getAttributeValue( + org.hisp.dhis.importexport.dxf2.model.DataValue.ATTR_VALUE ); + + DataValue dv = new DataValue(); + dv.setPeriod( outerPeriod ); + dv.setValue( value ); + // populate with placeholders + dv.setDataElement( new DataElement() ); + dv.setSource( new OrganisationUnit() ); + dv.setOptionCombo( optioncombo ); + dv.setComment( comment ); + dv.setStoredBy( user ); + dv.setTimestamp( timestamp ); + + // if no outer orgunit defined, use thae map + if ( outerOrgunit == null ) + { + dv.getSource().setId( orgunitMap.get( innerOrgUnitId ) ); + } else + { + dv.getSource().setId( outerOrgunitId ); + } + + dv.getDataElement().setId( dataelementMap.get( dataElementId ) ); + + batchHandler.addObject( dv ); + + ++countDataValues; + ++cumulativeDataValueCounter; + + log.debug( cumulativeDataValueCounter + " DataValues read" ); + + } while ( true ); // DataValues loop + + ++countDataValueSets; + log.debug( countDataValueSets + " DataValueSets read" ); + + } while ( true ); // DataValueSets loop + + } catch ( ImportException ex ) + { + log.warn( ex.toString()); + state.setMessage( ex.toString() ); + } finally + { + batchHandler.flush(); + } + + log.info( SUCCESS ); + state.setMessage( SUCCESS ); + } + + private Period getPeriodObj( String period ) + throws ImportException + { + Period periodObj; + + PeriodType pt = PeriodType.getPeriodTypeFromIsoString( period ); + + + if ( pt == null ) + { + throw new ImportException( String.format( INVALID_PERIOD, period ) ); + } + + try + { + periodObj = pt.createPeriod( period ); + } catch ( Exception e ) + { + throw new ImportException( String.format( INVALID_PERIOD, period ) ); + } + + return periodObj; + } + + private Map getDataElementMap( String dataSet, DataValueSet.IdentificationStrategy idScheme ) + { + Collection dataelements; + Map result = null; + + if ( dataSet != null ) + { + DataSet ds = getDataSet( dataSet, idScheme ); + dataelements = ds.getDataElements(); + } else + { + dataelements = dataElementService.getAllDataElements(); + } + switch ( idScheme ) + { + case UUID: + result = DataElement.getUUIDMap( dataelements ); + break; + case CODE: + result = DataElement.getCodeMap( dataelements ); + break; + case INTERNAL: + break; + default: + throw new IllegalArgumentException( "Can't map with :" + idScheme ); + } + return result; + } + + private Map getOrgUnitMap( String dataSet, DataValueSet.IdentificationStrategy idScheme ) + { + Collection orgunits; + Map result = null; + + if ( dataSet != null ) + { + DataSet ds = getDataSet( dataSet, idScheme ); + orgunits = ds.getSources(); + } else + { + orgunits = organisationUnitService.getAllOrganisationUnits(); + } + + switch ( idScheme ) + { + case UUID: + result = OrganisationUnit.getUUIDMap( orgunits ); + break; + case CODE: + result = OrganisationUnit.getCodeMap( orgunits ); + break; + case INTERNAL: + break; + default: + throw new IllegalArgumentException( "Can't map with :" + idScheme ); + } + return result; + } + + /** + * For a given orgunit identifier and id scheme, returns the orgunit object reference + * @param orgunit + * @param idScheme + * @return + * @throws ImportException thrown if no orgunit matches + */ + private OrganisationUnit getOrgUnitByIdentifier( String orgunit, DataValueSet.IdentificationStrategy idScheme ) + throws ImportException + { + OrganisationUnit ou; + switch ( idScheme ) + { + case UUID: + ou = organisationUnitService.getOrganisationUnit( orgunit ); + break; + case CODE: + ou = organisationUnitService.getOrganisationUnitByCode( orgunit ); + break; + case INTERNAL: + ou = organisationUnitService.getOrganisationUnit( Integer.parseInt( orgunit ) ); + break; + default: + throw new IllegalArgumentException( "Can't map with :" + idScheme ); + } + + if ( ou == null ) + { + throw new ImportException( String.format( NO_SUCH_ORGUNIT, idScheme, orgunit ) ); + } + return ou; + } + + /** + * For a given dataelement identifier and id scheme, returns the dataelement object reference + * @param dataelement + * @param idScheme + * @return + * @throws ImportException thrown if no dataelement matches + */ + private DataElement getDataElementByIdentifier( String dataelement, DataValueSet.IdentificationStrategy idScheme ) + throws ImportException + { + DataElement de; + switch ( idScheme ) + { + case UUID: + de = dataElementService.getDataElement( dataelement ); + break; + case CODE: + de = dataElementService.getDataElementByCode( dataelement ); + break; + case INTERNAL: + de = dataElementService.getDataElement( Integer.parseInt( dataelement ) ); + break; + default: + throw new IllegalArgumentException( "Can't map with :" + idScheme ); + } + + if ( de == null ) + { + throw new ImportException( String.format( NO_SUCH_DATAELEMENT, idScheme, dataelement ) ); + } + return de; + } + + private DataSet getDataSet( String dataSet, DataValueSet.IdentificationStrategy idScheme ) + { + DataSet result = null; + switch ( idScheme ) + { + case INTERNAL: + result = dataSetService.getDataSet( Integer.parseInt( dataSet ) ); + break; + case CODE: + result = dataSetService.getDataSetByCode( dataSet ); + break; + default: + result = dataSetService.getDataSet( dataSet ); + } + return result; + } +}