=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-11-18 17:13:43 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-12-03 07:07:47 +0000 @@ -28,25 +28,6 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PipedOutputStream; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamWriter; - import org.amplecode.staxwax.factory.XMLFactory; import org.amplecode.staxwax.reader.XMLReader; import org.apache.commons.lang3.StringUtils; @@ -79,6 +60,24 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PipedOutputStream; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + /** * @author bobj */ @@ -111,7 +110,7 @@ @Autowired private IdentifiableObjectManager identifiableObjectManager; - + @Autowired private Notifier notifier; @@ -130,7 +129,7 @@ public ImportSummaries saveDataValueSet( InputStream in, ImportOptions importOptions, TaskId id ) { notifier.clear( id ).notify( id, "ADX parsing process started" ); - + XMLReader adxReader = XMLFactory.getXMLReader( in ); ImportSummaries importSummaries = new ImportSummaries(); @@ -140,7 +139,7 @@ ExecutorService executor = Executors.newSingleThreadExecutor(); int count = 0; - + // submit each ADX group to DXF importer as a datavalueSet while ( adxReader.moveToStartElement( AdxDataService.GROUP, AdxDataService.NAMESPACE ) ) { @@ -150,24 +149,24 @@ futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, id, pipeOut ) ); XMLOutputFactory factory = XMLOutputFactory.newInstance(); XMLStreamWriter dxfWriter = factory.createXMLStreamWriter( pipeOut ); - + // note this returns conflicts which are detected at ADX level List adxConflicts = parseAdxGroupToDxf( adxReader, dxfWriter, importOptions ); - + pipeOut.flush(); - + ImportSummary summary = futureImportSummary.get( TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES ); - + // add ADX conflicts to the import summary - for ( ImportConflict conflict : adxConflicts) + for ( ImportConflict conflict : adxConflicts ) { summary.getConflicts().add( conflict ); summary.getImportCount().incrementIgnored(); } - + importSummaries.addImportSummary( summary ); } - catch ( AdxException ex) + catch ( AdxException ex ) { ImportSummary importSummary = new ImportSummary(); importSummary.setStatus( ImportStatus.ERROR ); @@ -184,12 +183,12 @@ importSummaries.addImportSummary( importSummary ); log.warn( "Import failed: " + ex ); } - + count++; } executor.shutdown(); - + return importSummaries; } @@ -201,12 +200,12 @@ throws XMLStreamException, AdxException { List adxConflicts = new LinkedList<>(); - + dxfWriter.writeStartDocument( "1.0" ); dxfWriter.writeStartElement( "dataValueSet" ); dxfWriter.writeDefaultNamespace( "http://dhis2.org/schema/dxf/2.0" ); - IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme(); + IdentifiableProperty dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme(); Map groupAttributes = adxReader.readAttributes(); @@ -224,7 +223,7 @@ String periodStr = groupAttributes.get( AdxDataService.PERIOD ); groupAttributes.remove( AdxDataService.PERIOD ); Period period = AdxPeriod.parse( periodStr ); - groupAttributes.put( AdxDataService.PERIOD, period.getIsoDate()); + groupAttributes.put( AdxDataService.PERIOD, period.getIsoDate() ); // process ADX group attributes if ( !groupAttributes.containsKey( AdxDataService.ATTOPTCOMBO ) @@ -234,12 +233,12 @@ DataSet dataSet = identifiableObjectManager.getObject( DataSet.class, dataElementIdScheme, groupAttributes.get( AdxDataService.DATASET ) ); - + if ( dataSet == null ) { - throw new AdxException("No data set matching identifier: " + groupAttributes.get( AdxDataService.DATASET ) ); + throw new AdxException( "No data set matching identifier: " + groupAttributes.get( AdxDataService.DATASET ) ); } - + groupAttributes.put( AdxDataService.DATASET, dataSet.getUid() ); DataElementCategoryCombo attributeCombo = dataSet.getCategoryCombo(); attributesToDxf( AdxDataService.ATTOPTCOMBO, attributeCombo, groupAttributes, dataElementIdScheme ); @@ -254,21 +253,21 @@ // process the dataValues while ( adxReader.moveToStartElement( AdxDataService.DATAVALUE, AdxDataService.GROUP ) ) { - try + try { parseADXDataValueToDxf( adxReader, dxfWriter, importOptions ); } - catch (AdxException ex) + catch ( AdxException ex ) { adxConflicts.add( ex.getImportConflict() ); - - log.info("ADX data value conflict: " + ex.getImportConflict() ); + + log.info( "ADX data value conflict: " + ex.getImportConflict() ); } } dxfWriter.writeEndElement(); dxfWriter.writeEndDocument(); - + return adxConflicts; } @@ -276,9 +275,9 @@ throws XMLStreamException, AdxException { Map dvAttributes = adxReader.readAttributes(); - + log.debug( "Processing data value: " + dvAttributes ); - + if ( !dvAttributes.containsKey( AdxDataService.DATAELEMENT ) ) { throw new AdxException( AdxDataService.DATAELEMENT + " attribute is required on 'dataValue'" ); @@ -289,30 +288,30 @@ throw new AdxException( AdxDataService.VALUE + " attribute is required on 'dataValue'" ); } - IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme(); + IdentifiableProperty dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme(); DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme, dvAttributes.get( AdxDataService.DATAELEMENT ) ); - + if ( dataElement == null ) { - throw new AdxException(dvAttributes.get( AdxDataService.DATAELEMENT), "No matching dataelement" ); + throw new AdxException( dvAttributes.get( AdxDataService.DATAELEMENT ), "No matching dataElement" ); } - + // process ADX data value attributes if ( !dvAttributes.containsKey( AdxDataService.CATOPTCOMBO ) ) { log.debug( "No category option combo present" ); - + DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo(); attributesToDxf( AdxDataService.CATOPTCOMBO, categoryCombo, dvAttributes, dataElementIdScheme ); } - + // if data element type is not numeric we need to pick out the 'annotation' element - if ( !dataElement.getValueType().isNumeric() ) + if ( !dataElement.getValueType().isNumeric() ) { adxReader.moveToStartElement( AdxDataService.ANNOTATION, AdxDataService.DATAVALUE ); - + if ( adxReader.isStartElement( AdxDataService.ANNOTATION ) ) { String textValue = adxReader.getElementValue(); @@ -320,20 +319,20 @@ } else { - throw new AdxException( dvAttributes.get( AdxDataService.DATAELEMENT),"Dataelement expects text annotation" ); + throw new AdxException( dvAttributes.get( AdxDataService.DATAELEMENT ), "DataElement expects text annotation" ); } } - + log.debug( "Processing data value as DXF: " + dvAttributes ); - + dxfWriter.writeStartElement( "dataValue" ); - + // pass through the remaining attributes to DXF for ( String attribute : dvAttributes.keySet() ) { dxfWriter.writeAttribute( attribute, dvAttributes.get( attribute ) ); } - + dxfWriter.writeEndElement(); } @@ -347,13 +346,13 @@ for ( DataElementCategory category : categories ) { String categoryCode = category.getCode(); - + if ( categoryCode == null || !XMLChar.isValidName( categoryCode ) ) { - throw new AdxException( "Category code for " + category.getName() + + throw new AdxException( "Category code for " + category.getName() + " is missing or invalid: " + categoryCode ); } - + categoryMap.put( category.getCode(), category ); } @@ -365,7 +364,7 @@ throws AdxException { CategoryComboMap catcomboMap; - + try { catcomboMap = new CategoryComboMap( catcombo, scheme ); @@ -379,33 +378,33 @@ } String compositeIdentifier = StringUtils.EMPTY; - + for ( DataElementCategory category : catcomboMap.getCategories() ) { String categoryCode = category.getCode(); - + if ( categoryCode == null ) { throw new AdxException( "No category matching: " + categoryCode ); } - + String catAttribute = attributes.get( categoryCode ); - + if ( catAttribute == null ) { throw new AdxException( "Missing required attribute from category combo: " + categoryCode ); } - + compositeIdentifier += "\"" + catAttribute + "\""; } - + DataElementCategoryOptionCombo catOptionCombo = catcomboMap.getCategoryOptionCombo( compositeIdentifier ); if ( catOptionCombo == null ) { throw new AdxException( "Invalid attributes:" + attributes ); } - + return catOptionCombo; } @@ -419,11 +418,11 @@ { return; } - + Map categoryMap = createCategoryMap( catCombo ); Map attributeOptions = new HashMap<>(); - + for ( String category : categoryMap.keySet() ) { if ( attributes.containsKey( category ) ) @@ -438,9 +437,9 @@ } DataElementCategoryOptionCombo catOptCombo = getCatOptComboFromAttributes( attributeOptions, catCombo, scheme ); - + attributes.put( optionComboName, catOptCombo.getUid() ); - + log.debug( "DXF attributes: " + attributes ); } } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/IdSchemes.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/IdSchemes.java 2015-12-03 02:17:45 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/IdSchemes.java 2015-12-03 07:07:47 +0000 @@ -38,7 +38,7 @@ */ public class IdSchemes { - private IdentifiableProperty idScheme; + private IdentifiableProperty idScheme = IdentifiableProperty.UID; private String idSchemeAttribute; @@ -81,16 +81,17 @@ return idSchemeAttribute; } - public void setIdScheme( String idScheme ) + public IdSchemes setIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.idScheme = IdentifiableProperty.ATTRIBUTE; this.idSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.idScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public IdentifiableProperty getDataElementIdScheme() @@ -103,16 +104,17 @@ return dataElementIdSchemeAttribute; } - public void setDataElementIdScheme( String idScheme ) + public IdSchemes setDataElementIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.dataElementIdScheme = IdentifiableProperty.ATTRIBUTE; this.dataElementIdSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.dataElementIdScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public IdentifiableProperty getCategoryOptionComboIdScheme() @@ -125,16 +127,17 @@ return categoryOptionComboIdSchemeAttribute; } - public void setCategoryOptionComboIdScheme( String idScheme ) + public IdSchemes setCategoryOptionComboIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.categoryOptionComboIdScheme = IdentifiableProperty.ATTRIBUTE; this.categoryOptionComboIdSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.categoryOptionComboIdScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public IdentifiableProperty getOrgUnitIdScheme() @@ -147,16 +150,17 @@ return orgUnitIdSchemeAttribute; } - public void setOrgUnitIdScheme( String idScheme ) + public IdSchemes setOrgUnitIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.orgUnitIdScheme = IdentifiableProperty.ATTRIBUTE; this.orgUnitIdSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.orgUnitIdScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public IdentifiableProperty getProgramIdScheme() @@ -169,16 +173,17 @@ return programIdSchemeAttribute; } - public void setProgramIdScheme( String idScheme ) + public IdSchemes setProgramIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.programIdScheme = IdentifiableProperty.ATTRIBUTE; this.programIdSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.programIdScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public IdentifiableProperty getProgramStageIdScheme() @@ -191,16 +196,17 @@ return programStageIdSchemeAttribute; } - public void setProgramStageIdScheme( String idScheme ) + public IdSchemes setProgramStageIdScheme( String idScheme ) { if ( isAttribute( idScheme ) ) { this.programStageIdScheme = IdentifiableProperty.ATTRIBUTE; this.programStageIdSchemeAttribute = idScheme.substring( 10 ); - return; + return this; } this.programStageIdScheme = IdentifiableProperty.valueOf( idScheme.toUpperCase() ); + return this; } public static String getValue( String uid, String code, IdentifiableProperty identifiableProperty ) === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java 2015-08-31 08:31:30 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/common/ImportOptions.java 2015-12-03 07:07:47 +0000 @@ -28,14 +28,10 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import static org.hisp.dhis.common.IdentifiableProperty.UID; - -import org.hisp.dhis.common.IdentifiableProperty; +import com.google.common.base.MoreObjects; import org.hisp.dhis.common.MergeStrategy; import org.hisp.dhis.importexport.ImportStrategy; -import com.google.common.base.MoreObjects; - /** * The idScheme is a general setting which will apply to all objects. The idSchemes * can also be defined for specific objects such as dataElementIdScheme. The @@ -45,14 +41,9 @@ */ public class ImportOptions { - private static final ImportOptions DEFAULT_OPTIONS = new ImportOptions(). - setDataElementIdScheme( UID ).setOrgUnitIdScheme( UID ).setImportStrategy( ImportStrategy.NEW_AND_UPDATES ); - - private IdentifiableProperty idScheme; - - private IdentifiableProperty dataElementIdScheme; - - private IdentifiableProperty orgUnitIdScheme; + private static final ImportOptions DEFAULT_OPTIONS = new ImportOptions().setImportStrategy( ImportStrategy.NEW_AND_UPDATES ); + + private IdSchemes idSchemes = new IdSchemes(); private boolean dryRun; @@ -67,19 +58,19 @@ private boolean skipExistingCheck; private boolean sharing; - + private boolean strictPeriods; private boolean strictCategoryOptionCombos; - + private boolean strictAttributeOptionCombos; - + private boolean strictOrganisationUnits; - + private boolean requireCategoryOptionCombo; - + private boolean requireAttributeOptionCombo; - + //-------------------------------------------------------------------------- // Constructors //-------------------------------------------------------------------------- @@ -87,13 +78,6 @@ public ImportOptions() { } - - public ImportOptions( IdentifiableProperty idScheme, IdentifiableProperty dataElementIdScheme, IdentifiableProperty orgUnitIdscheme ) - { - this.idScheme = idScheme; - this.dataElementIdScheme = dataElementIdScheme; - this.orgUnitIdScheme = orgUnitIdscheme; - } //-------------------------------------------------------------------------- // Logic @@ -103,24 +87,14 @@ { return DEFAULT_OPTIONS; } - + //-------------------------------------------------------------------------- // Get methods //-------------------------------------------------------------------------- - public IdentifiableProperty getIdScheme() - { - return idScheme != null ? idScheme : IdentifiableProperty.UID; - } - - public IdentifiableProperty getDataElementIdScheme() - { - return dataElementIdScheme != null ? dataElementIdScheme : ( idScheme != null ? idScheme : IdentifiableProperty.UID ); - } - - public IdentifiableProperty getOrgUnitIdScheme() - { - return orgUnitIdScheme != null ? orgUnitIdScheme : ( idScheme != null ? idScheme : IdentifiableProperty.UID ); + public IdSchemes getIdSchemes() + { + return idSchemes; } public boolean isDryRun() @@ -162,7 +136,7 @@ { return sharing; } - + public boolean isStrictPeriods() { return strictPeriods; @@ -197,22 +171,75 @@ // Set methods //-------------------------------------------------------------------------- - public ImportOptions setIdScheme( IdentifiableProperty scheme ) - { - this.idScheme = scheme != null ? scheme : null; - return this; - } - - public ImportOptions setDataElementIdScheme( IdentifiableProperty scheme ) - { - this.dataElementIdScheme = scheme != null ? scheme : null; - return this; - } - - public ImportOptions setOrgUnitIdScheme( IdentifiableProperty scheme ) - { - this.orgUnitIdScheme = scheme != null ? scheme : null; - return this; + public ImportOptions setProgramStageIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setProgramStageIdScheme( idScheme ); + return this; + } + + public ImportOptions setProgramIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setProgramIdScheme( idScheme ); + return this; + } + + public ImportOptions setOrgUnitIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setOrgUnitIdScheme( idScheme ); + return this; + } + + public ImportOptions setCategoryOptionComboIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setCategoryOptionComboIdScheme( idScheme ); + return this; + } + + public ImportOptions setDataElementIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setDataElementIdScheme( idScheme ); + return this; + } + + public ImportOptions setIdScheme( String idScheme ) + { + if ( this.idSchemes == null ) + { + this.idSchemes = new IdSchemes(); + } + + idSchemes.setIdScheme( idScheme ); + return this; + } + + public void setIdSchemes( IdSchemes idSchemes ) + { + this.idSchemes = idSchemes; } public ImportOptions setDryRun( boolean dryRun ) @@ -297,9 +324,7 @@ public String toString() { return MoreObjects.toStringHelper( this.getClass() ). - add( "Id scheme", idScheme ). - add( "Data element id scheme", dataElementIdScheme ). - add( "Org unit id scheme", orgUnitIdScheme ). + add( "ID Schemes", idSchemes ). add( "Dry run", dryRun ). add( "Preheat cache", preheatCache ). add( "Async", async ). === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2015-11-30 17:16:26 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2015-12-03 07:07:47 +0000 @@ -28,24 +28,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import static org.apache.commons.lang3.StringUtils.trimToNull; -import static org.hisp.dhis.common.IdentifiableProperty.UUID; -import static org.hisp.dhis.system.notification.NotificationLevel.ERROR; -import static org.hisp.dhis.system.notification.NotificationLevel.INFO; -import static org.hisp.dhis.system.util.DateUtils.getDefaultDate; -import static org.hisp.dhis.system.util.DateUtils.parseDate; - -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Writer; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - +import com.csvreader.CsvReader; import org.amplecode.quick.BatchHandler; import org.amplecode.quick.BatchHandlerFactory; import org.amplecode.staxwax.factory.XMLFactory; @@ -100,7 +83,23 @@ import org.hisp.dhis.user.CurrentUserService; import org.springframework.beans.factory.annotation.Autowired; -import com.csvreader.CsvReader; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Writer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.commons.lang3.StringUtils.trimToNull; +import static org.hisp.dhis.common.IdentifiableProperty.UUID; +import static org.hisp.dhis.system.notification.NotificationLevel.ERROR; +import static org.hisp.dhis.system.notification.NotificationLevel.INFO; +import static org.hisp.dhis.system.util.DateUtils.getDefaultDate; +import static org.hisp.dhis.system.util.DateUtils.parseDate; /** * @author Lars Helge Overland @@ -114,7 +113,7 @@ @Autowired private IdentifiableObjectManager identifiableObjectManager; - + @Autowired private DataElementService dataElementService; @@ -138,10 +137,10 @@ @Autowired private DataValueSetStore dataValueSetStore; - + @Autowired private SystemSettingManager systemSettingManager; - + @Autowired private I18nManager i18nManager; @@ -149,7 +148,7 @@ private Notifier notifier; // Set methods for test purposes - + public void setBatchHandlerFactory( BatchHandlerFactory batchHandlerFactory ) { this.batchHandlerFactory = batchHandlerFactory; @@ -165,16 +164,16 @@ //-------------------------------------------------------------------------- @Override - public DataExportParams getFromUrl( Set dataSets, Set periods, Date startDate, Date endDate, + public DataExportParams getFromUrl( Set dataSets, Set periods, Date startDate, Date endDate, Set organisationUnits, boolean includeChildren, Date lastUpdated, Integer limit, IdSchemes idSchemes ) { DataExportParams params = new DataExportParams(); - + if ( dataSets != null ) { params.getDataSets().addAll( identifiableObjectManager.getByUid( DataSet.class, dataSets ) ); } - + if ( periods != null && !periods.isEmpty() ) { params.getPeriods().addAll( periodService.reloadIsoPeriods( new ArrayList<>( periods ) ) ); @@ -184,7 +183,7 @@ params.setStartDate( startDate ); params.setEndDate( endDate ); } - + if ( organisationUnits != null ) { params.getOrganisationUnits().addAll( identifiableObjectManager.getByUid( OrganisationUnit.class, organisationUnits ) ); @@ -194,49 +193,49 @@ params.setLastUpdated( lastUpdated ); params.setLimit( limit ); params.setIdSchemes( idSchemes ); - + return params; } - + @Override public void validate( DataExportParams params ) { String violation = null; - + if ( params == null ) { throw new IllegalArgumentException( "Params cannot be null" ); } - + if ( params.getDataSets().isEmpty() ) { violation = "At least one valid data set must be specified"; } - - if ( params.getPeriods().isEmpty() && !params.hasStartEndDate() ) + + if ( params.getPeriods().isEmpty() && !params.hasStartEndDate() ) { violation = "At least one valid period or start/end dates must be specified"; } - + if ( params.hasStartEndDate() && params.getStartDate().after( params.getEndDate() ) ) { violation = "Start date must be before end date"; } - + if ( params.getOrganisationUnits().isEmpty() ) { violation = "At least one valid organisation unit must be specified"; } - + if ( params.hasLimit() && params.getLimit() < 0 ) { violation = "Limit cannot be less than zero: " + params.getLimit(); } - + if ( violation != null ) { log.warn( "Validation failed: " + violation ); - + throw new IllegalArgumentException( violation ); } } @@ -252,7 +251,7 @@ } } } - + //-------------------------------------------------------------------------- // Write //-------------------------------------------------------------------------- @@ -265,7 +264,7 @@ dataValueSetStore.writeDataValueSetXml( params, getCompleteDate( params ), out ); } - + @Override public void writeDataValueSetJson( DataExportParams params, OutputStream out ) { @@ -277,7 +276,7 @@ @Override public void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream, IdSchemes idSchemes ) - { + { dataValueSetStore.writeDataValueSetJson( lastUpdated, outputStream, idSchemes ); } @@ -286,7 +285,7 @@ { decideAccess( params ); validate( params ); - + dataValueSetStore.writeDataValueSetCsv( params, getCompleteDate( params ), writer ); } @@ -295,13 +294,13 @@ if ( params.isSingleDataValueSet() ) { DataElementCategoryOptionCombo optionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); //TODO - + CompleteDataSetRegistration registration = registrationService .getCompleteDataSetRegistration( params.getFirstDataSet(), params.getFirstPeriod(), params.getFirstOrganisationUnit(), optionCombo ); - + return registration != null ? registration.getDate() : null; } - + return null; } @@ -516,15 +515,15 @@ * a generic id scheme for all objects. The specific id schemes will take * precedence over the generic id scheme. The generic id scheme also applies * to data set and category option combo. - *

+ *

* The id schemes uses the following order of precedence: - *

+ *

*

    *
  • Id scheme from the data value set
    • *
  • Id scheme from the import options
    • *
  • Default id scheme which is UID
    • *
      - *

      + *

      * If id scheme is specific in the data value set, any id schemes in the import * options will be ignored. * @@ -537,7 +536,7 @@ { Clock clock = new Clock( log ).startClock().logTime( "Starting data value import, options: " + importOptions ); notifier.clear( id ).notify( id, "Process started" ); - + ImportSummary summary = new ImportSummary(); I18n i18n = i18nManager.getI18n(); @@ -556,9 +555,9 @@ log.info( "Data value set scheme: " + dvSetIdScheme + ", data element scheme: " + dvSetDataElementIdScheme + ", org unit scheme: " + dvSetOrgUnitIdScheme ); - IdentifiableProperty idScheme = dvSetIdScheme != null ? dvSetIdScheme : importOptions.getIdScheme(); - IdentifiableProperty dataElementIdScheme = dvSetDataElementIdScheme != null ? dvSetDataElementIdScheme : importOptions.getDataElementIdScheme(); - IdentifiableProperty orgUnitIdScheme = dvSetOrgUnitIdScheme != null ? dvSetOrgUnitIdScheme : importOptions.getOrgUnitIdScheme(); + IdentifiableProperty idScheme = dvSetIdScheme != null ? dvSetIdScheme : importOptions.getIdSchemes().getIdScheme(); + IdentifiableProperty dataElementIdScheme = dvSetDataElementIdScheme != null ? dvSetDataElementIdScheme : importOptions.getIdSchemes().getDataElementIdScheme(); + IdentifiableProperty orgUnitIdScheme = dvSetOrgUnitIdScheme != null ? dvSetOrgUnitIdScheme : importOptions.getIdSchemes().getOrgUnitIdScheme(); log.info( "Scheme: " + idScheme + ", data element scheme: " + dataElementIdScheme + ", org unit scheme: " + orgUnitIdScheme ); @@ -566,14 +565,14 @@ ImportStrategy.valueOf( dataValueSet.getStrategy() ) : importOptions.getImportStrategy(); boolean dryRun = dataValueSet.getDryRun() != null ? dataValueSet.getDryRun() : importOptions.isDryRun(); - boolean skipExistingCheck = importOptions.isSkipExistingCheck(); + boolean skipExistingCheck = importOptions.isSkipExistingCheck(); boolean strictPeriods = importOptions.isStrictPeriods() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_STRICT_PERIODS ); boolean strictCategoryOptionCombos = importOptions.isStrictCategoryOptionCombos() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS ); boolean strictAttrOptionCombos = importOptions.isStrictAttributeOptionCombos() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_STRICT_ATTRIBUTE_OPTION_COMBOS ); boolean strictOrgUnits = importOptions.isStrictOrganisationUnits() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_STRICT_ORGANISATION_UNITS ); boolean requireCategoryOptionCombo = importOptions.isRequireCategoryOptionCombo() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO ); boolean requireAttrOptionCombo = importOptions.isRequireAttributeOptionCombo() || (Boolean) systemSettingManager.getSystemSetting( Setting.DATA_IMPORT_REQUIRE_ATTRIBUTE_OPTION_COMBO ); - + //---------------------------------------------------------------------- // Create meta-data maps //---------------------------------------------------------------------- @@ -598,25 +597,25 @@ { notifier.notify( id, "Loading data elements and organisation units" ); dataElementMap.putAll( identifiableObjectManager.getIdMap( DataElement.class, dataElementIdScheme ) ); - orgUnitMap.putAll( getOrgUnitMap( orgUnitIdScheme ) ); + orgUnitMap.putAll( getOrgUnitMap( orgUnitIdScheme ) ); clock.logTime( "Preheated data element and organisation unit caches" ); } - - IdentifiableObjectCallable dataElementCallable = new IdentifiableObjectCallable<>( + + IdentifiableObjectCallable dataElementCallable = new IdentifiableObjectCallable<>( identifiableObjectManager, DataElement.class, dataElementIdScheme, null ); - IdentifiableObjectCallable orgUnitCallable = new IdentifiableObjectCallable<>( + IdentifiableObjectCallable orgUnitCallable = new IdentifiableObjectCallable<>( identifiableObjectManager, OrganisationUnit.class, orgUnitIdScheme, trimToNull( dataValueSet.getOrgUnit() ) ); - IdentifiableObjectCallable optionComboCallable = new CategoryOptionComboAclCallable( + IdentifiableObjectCallable optionComboCallable = new CategoryOptionComboAclCallable( categoryService, idScheme, null ); - IdentifiableObjectCallable periodCallable = new PeriodCallable( + IdentifiableObjectCallable periodCallable = new PeriodCallable( periodService, null, trimToNull( dataValueSet.getPeriod() ) ); - + //---------------------------------------------------------------------- // Get outer meta-data //---------------------------------------------------------------------- DataSet dataSet = dataValueSet.getDataSet() != null ? identifiableObjectManager.getObject( DataSet.class, idScheme, dataValueSet.getDataSet() ) : null; - + Date completeDate = getDefaultDate( dataValueSet.getCompleteDate() ); Period outerPeriod = periodMap.get( trimToNull( dataValueSet.getPeriod() ), periodCallable ); @@ -625,7 +624,7 @@ DataElementCategoryOptionCombo fallbackCategoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); - DataElementCategoryOptionCombo outerAttrOptionCombo = dataValueSet.getAttributeOptionCombo() != null ? + DataElementCategoryOptionCombo outerAttrOptionCombo = dataValueSet.getAttributeOptionCombo() != null ? optionComboMap.get( trimToNull( dataValueSet.getAttributeOptionCombo() ), optionComboCallable.setId( trimToNull( dataValueSet.getAttributeOptionCombo() ) ) ) : null; // --------------------------------------------------------------------- @@ -670,7 +669,7 @@ final String currentUser = currentUserService.getCurrentUsername(); final Set currentOrgUnits = currentUserService.getCurrentUserOrganisationUnits(); - + BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataValueBatchHandler.class ).init(); int importCount = 0; @@ -692,13 +691,13 @@ totalCount++; - final DataElement dataElement = + final DataElement dataElement = dataElementMap.get( trimToNull( dataValue.getDataElement() ), dataElementCallable.setId( trimToNull( dataValue.getDataElement() ) ) ); - final Period period = outerPeriod != null ? outerPeriod : + final Period period = outerPeriod != null ? outerPeriod : periodMap.get( trimToNull( dataValue.getPeriod() ), periodCallable.setId( trimToNull( dataValue.getPeriod() ) ) ); - final OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : + final OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : orgUnitMap.get( trimToNull( dataValue.getOrgUnit() ), orgUnitCallable.setId( trimToNull( dataValue.getOrgUnit() ) ) ); - DataElementCategoryOptionCombo categoryOptionCombo = optionComboMap.get( trimToNull( dataValue.getCategoryOptionCombo() ), + DataElementCategoryOptionCombo categoryOptionCombo = optionComboMap.get( trimToNull( dataValue.getCategoryOptionCombo() ), optionComboCallable.setId( trimToNull( dataValue.getCategoryOptionCombo() ) ) ); DataElementCategoryOptionCombo attrOptionCombo = outerAttrOptionCombo != null ? outerAttrOptionCombo : optionComboMap.get( trimToNull( dataValue.getAttributeOptionCombo() ), optionComboCallable.setId( trimToNull( dataValue.getAttributeOptionCombo() ) ) ); @@ -718,7 +717,7 @@ summary.getConflicts().add( new ImportConflict( dataValue.getPeriod(), "Period not valid" ) ); continue; } - + if ( orgUnit == null ) { summary.getConflicts().add( new ImportConflict( dataValue.getOrgUnit(), "Organisation unit not found or not accessible" ) ); @@ -737,18 +736,18 @@ continue; } - boolean inUserHierarchy = orgUnitInHierarchyMap.get( orgUnit.getUid(), + boolean inUserHierarchy = orgUnitInHierarchyMap.get( orgUnit.getUid(), () -> organisationUnitService.isInUserHierarchy( orgUnit.getUid(), currentOrgUnits ) ); - + if ( !inUserHierarchy ) { summary.getConflicts().add( new ImportConflict( orgUnit.getUid(), "Organisation unit not in hierarchy of current user: " + currentUser ) ); continue; } - + boolean invalidFuturePeriod = period.isFuture() && dataElementOpenFuturePeriodsMap.get( dataElement.getUid(), () -> dataElementService.isOpenFuturePeriods( dataElement.getId() ) ); - + if ( invalidFuturePeriod ) { summary.getConflicts().add( new ImportConflict( period.getIsoDate(), "Data element does not allow for future periods through data sets: " + dataElement.getUid() ) ); @@ -776,15 +775,15 @@ continue; } - Optional> optionCodes = dataElementOptionsMap.get( dataElement.getUid(), + Optional> optionCodes = dataElementOptionsMap.get( dataElement.getUid(), () -> dataElementService.getOptionCodesAsSet( dataElement.getId() ) ); - + if ( optionCodes.isPresent() && !optionCodes.get().contains( dataValue.getValue() ) ) { summary.getConflicts().add( new ImportConflict( dataValue.getValue(), "Data value is not a valid option of the data element option set: " + dataElement.getUid() ) ); continue; } - + // ----------------------------------------------------------------- // Constraints // ----------------------------------------------------------------- @@ -814,23 +813,23 @@ attrOptionCombo = fallbackCategoryOptionCombo; } } - - if ( strictPeriods && !dataElementPeriodTypesMap.get( dataElement.getUid(), + + if ( strictPeriods && !dataElementPeriodTypesMap.get( dataElement.getUid(), () -> dataElement.getPeriodTypes() ).contains( period.getPeriodType() ) ) { - summary.getConflicts().add( new ImportConflict( dataValue.getPeriod(), + summary.getConflicts().add( new ImportConflict( dataValue.getPeriod(), "Period type of period: " + period.getIsoDate() + " not valid for data element: " + dataElement.getUid() ) ); continue; } - + if ( strictCategoryOptionCombos && !dataElementCategoryOptionComboMap.get( dataElement.getUid(), () -> dataElement.getCategoryCombo().getOptionCombos() ).contains( categoryOptionCombo ) ) { - summary.getConflicts().add( new ImportConflict( categoryOptionCombo.getUid(), + summary.getConflicts().add( new ImportConflict( categoryOptionCombo.getUid(), "Category option combo: " + categoryOptionCombo.getUid() + " must be part of category combo of data element: " + dataElement.getUid() ) ); continue; } - + if ( strictAttrOptionCombos && !dataElementAttrOptionComboMap.get( dataElement.getUid(), () -> dataElement.getDataSetCategoryOptionCombos() ).contains( attrOptionCombo ) ) { @@ -838,7 +837,7 @@ "Attribute option combo: " + attrOptionCombo.getUid() + " must be part of category combo of data sets of data element: " + dataElement.getUid() ) ); continue; } - + if ( strictOrgUnits && BooleanUtils.isFalse( dataElementOrgUnitMap.get( dataElement.getUid() + orgUnit.getUid(), () -> dataElement.hasDataSetOrganisationUnit( orgUnit ) ) ) ) { @@ -862,7 +861,7 @@ summary.getConflicts().add( new ImportConflict( dataValue.getStoredBy(), i18n.getString( storedByValid ) ) ); continue; } - + String storedBy = dataValue.getStoredBy() == null || dataValue.getStoredBy().trim().isEmpty() ? currentUser : dataValue.getStoredBy(); // ----------------------------------------------------------------- @@ -983,8 +982,8 @@ private Map getOrgUnitMap( IdentifiableProperty orgUnitIdScheme ) { - return UUID.equals( orgUnitIdScheme ) ? - organisationUnitService.getUuidOrganisationUnitMap() : - identifiableObjectManager.getIdMap( OrganisationUnit.class, orgUnitIdScheme ); + return UUID.equals( orgUnitIdScheme ) ? + organisationUnitService.getUuidOrganisationUnitMap() : + identifiableObjectManager.getIdMap( OrganisationUnit.class, orgUnitIdScheme ); } } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java 2015-12-01 13:29:32 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/AbstractEventService.java 2015-12-03 07:07:47 +0000 @@ -30,7 +30,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -398,7 +397,7 @@ } } - OrganisationUnit organisationUnit = getOrganisationUnit( importOptions.getOrgUnitIdScheme(), event.getOrgUnit() ); + OrganisationUnit organisationUnit = getOrganisationUnit( importOptions.getIdSchemes().getOrgUnitIdScheme(), event.getOrgUnit() ); if ( organisationUnit == null ) { @@ -511,7 +510,7 @@ EventStatus status, Date lastUpdated, DataElementCategoryOptionCombo attributeCoc, IdSchemes idSchemes, Integer page, Integer pageSize, boolean totalPages, boolean skipPaging, boolean includeAttributes ) { UserCredentials userCredentials = currentUserService.getCurrentUser().getUserCredentials(); - + EventSearchParams params = new EventSearchParams(); Program pr = programService.getProgram( program ); @@ -534,24 +533,24 @@ { throw new IllegalQueryException( "Org unit is specified but does not exist: " + orgUnit ); } - - if( ou != null && !organisationUnitService.isInUserHierarchy( ou ) ) - { - if( !userCredentials.isSuper() && !userCredentials.isAuthorized( "F_TRACKED_ENTITY_INSTANCE_SEARCH_IN_ALL_ORGUNITS" ) ) + + if ( ou != null && !organisationUnitService.isInUserHierarchy( ou ) ) + { + if ( !userCredentials.isSuper() && !userCredentials.isAuthorized( "F_TRACKED_ENTITY_INSTANCE_SEARCH_IN_ALL_ORGUNITS" ) ) { throw new IllegalQueryException( "User has no access to organisation unit: " + ou.getUid() ); } } - - if( pr != null && !userCredentials.isSuper() && userCredentials.getAllPrograms().size() == 0 ) + + if ( pr != null && !userCredentials.isSuper() && userCredentials.getAllPrograms().size() == 0 ) { - throw new IllegalQueryException( "User has no access to programs"); + throw new IllegalQueryException( "User has no access to programs" ); } - - if( pr != null && !userCredentials.getAllPrograms().contains( pr ) ) + + if ( pr != null && !userCredentials.getAllPrograms().contains( pr ) ) { throw new IllegalQueryException( "User has no access to program: " + pr.getUid() ); - } + } TrackedEntityInstance tei = entityInstanceService.getTrackedEntityInstance( trackedEntityInstance ); @@ -662,7 +661,7 @@ importOptions = new ImportOptions(); } - OrganisationUnit organisationUnit = getOrganisationUnit( importOptions.getOrgUnitIdScheme(), event.getOrgUnit() ); + OrganisationUnit organisationUnit = getOrganisationUnit( importOptions.getIdSchemes().getOrgUnitIdScheme(), event.getOrgUnit() ); if ( organisationUnit == null ) { @@ -737,9 +736,9 @@ Set dataValues = new HashSet<>( dataValueService.getTrackedEntityDataValues( programStageInstance ) ); Map existingDataValues = getDataElementDataValueMap( dataValues ); - + for ( DataValue value : event.getDataValues() ) - { + { DataElement dataElement = getDataElement( value.getDataElement() ); TrackedEntityDataValue dataValue = dataValueService.getTrackedEntityDataValue( programStageInstance, dataElement ); @@ -750,11 +749,11 @@ if ( dataValue != null ) { - if ( StringUtils.isEmpty( value.getValue() ) && dataElement.isFileType() && !StringUtils.isEmpty( dataValue.getValue() ) ) + if ( StringUtils.isEmpty( value.getValue() ) && dataElement.isFileType() && !StringUtils.isEmpty( dataValue.getValue() ) ) { - fileResourceService.deleteFileResource( dataValue.getValue() ); + fileResourceService.deleteFileResource( dataValue.getValue() ); } - + dataValue.setValue( value.getValue() ); dataValue.setProvidedElsewhere( value.getProvidedElsewhere() ); dataValueService.updateTrackedEntityDataValue( dataValue ); @@ -897,30 +896,30 @@ event.setCompletedDate( DateUtils.getLongDateString( programStageInstance.getCompletedDate() ) ); UserCredentials userCredentials = currentUserService.getCurrentUser().getUserCredentials(); - + OrganisationUnit ou = programStageInstance.getOrganisationUnit(); - + if ( ou != null ) - { - if( !organisationUnitService.isInUserHierarchy( ou ) ) - { - if( !userCredentials.isSuper() && !userCredentials.isAuthorized( "F_TRACKED_ENTITY_INSTANCE_SEARCH_IN_ALL_ORGUNITS" ) ) + { + if ( !organisationUnitService.isInUserHierarchy( ou ) ) + { + if ( !userCredentials.isSuper() && !userCredentials.isAuthorized( "F_TRACKED_ENTITY_INSTANCE_SEARCH_IN_ALL_ORGUNITS" ) ) { throw new IllegalQueryException( "User has no access to organisation unit: " + ou.getUid() ); } } - + event.setOrgUnit( ou.getUid() ); } - + Program program = programStageInstance.getProgramInstance().getProgram(); - - if( !userCredentials.isSuper() && !userCredentials.getAllPrograms().contains( program ) ) + + if ( !userCredentials.isSuper() && !userCredentials.getAllPrograms().contains( program ) ) { throw new IllegalQueryException( "User has no access to program: " + program.getUid() ); } - - event.setProgram( program.getUid() ); + + event.setProgram( program.getUid() ); event.setEnrollment( programStageInstance.getProgramInstance().getUid() ); event.setProgramStage( programStageInstance.getProgramStage().getUid() ); @@ -1329,5 +1328,5 @@ private DataElement getDataElement( String dataElementId ) { return dataElementCache.get( dataElementId, new IdentifiableObjectCallable<>( manager, DataElement.class, dataElementId ) ); - } + } } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java 2015-11-30 17:16:26 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java 2015-12-03 07:07:47 +0000 @@ -28,15 +28,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import static org.hisp.dhis.common.IdentifiableProperty.CODE; -import static org.hisp.dhis.common.IdentifiableProperty.UID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.InputStream; -import java.util.Collection; - +import com.google.common.collect.Sets; import org.hisp.dhis.DhisSpringTest; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.dataelement.DataElement; @@ -51,6 +43,7 @@ import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.dataset.DataSetService; import org.hisp.dhis.datavalue.DataValue; +import org.hisp.dhis.dxf2.common.IdSchemes; import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.importsummary.ImportStatus; import org.hisp.dhis.dxf2.importsummary.ImportSummary; @@ -73,7 +66,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; -import com.google.common.collect.Sets; +import java.io.InputStream; +import java.util.Collection; + +import static org.junit.Assert.*; /** * @author Lars Helge Overland @@ -83,25 +79,25 @@ { @Autowired private DataElementService dataElementService; - + @Autowired private DataElementCategoryService categoryService; - + @Autowired private OrganisationUnitService organisationUnitService; - + @Autowired private DataSetService dataSetService; - + @Autowired private PeriodService periodService; - + @Autowired private DataValueSetService dataValueSetService; - + @Autowired private CompleteDataSetRegistrationService registrationService; - + @Autowired private IdentifiableObjectManager idObjectManager; @@ -126,14 +122,14 @@ private OrganisationUnit ouC; private Period peA; private Period peB; - + private User user; - + private InputStream in; private MockBatchHandler mockDataValueBatchHandler = null; private MockBatchHandlerFactory mockBatchHandlerFactory = null; - + @Override public void setUpTest() { @@ -153,7 +149,7 @@ osA.getOptions().add( new Option( "Blue", "1" ) ); osA.getOptions().add( new Option( "Green", "2" ) ); osA.getOptions().add( new Option( "Yellow", "3" ) ); - + ocA = createCategoryOptionCombo( categoryComboA, categoryOptionA ); ocB = createCategoryOptionCombo( categoryComboA, categoryOptionB ); deA = createDataElement( 'A', categoryComboDef ); @@ -198,30 +194,30 @@ categoryService.addDataElementCategoryCombo( categoryComboA ); categoryService.addDataElementCategoryOptionCombo( ocA ); categoryService.addDataElementCategoryOptionCombo( ocB ); - + dataElementService.addDataElement( deA ); dataElementService.addDataElement( deB ); dataElementService.addDataElement( deC ); dataElementService.addDataElement( deD ); idObjectManager.save( osA ); - + dsA.addDataElement( deA ); dsA.addDataElement( deB ); dsA.addDataElement( deC ); - dsA.addDataElement( deD ); - + dsA.addDataElement( deD ); + organisationUnitService.addOrganisationUnit( ouA ); organisationUnitService.addOrganisationUnit( ouB ); organisationUnitService.addOrganisationUnit( ouC ); - + dsA.addOrganisationUnit( ouA ); dsA.addOrganisationUnit( ouC ); - + dataSetService.addDataSet( dsA ); periodService.addPeriod( peA ); periodService.addPeriod( peB ); - + user = createUser( 'A' ); user.setOrganisationUnits( Sets.newHashSet( ouA, ouB ) ); CurrentUserService currentUserService = new MockCurrentUserService( user ); @@ -237,24 +233,24 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetA.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); - + assertNotNull( summary ); assertNotNull( summary.getImportCount() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 3, dataValues.size() ); assertTrue( dataValues.contains( new DataValue( deA, peA, ouA, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deB, peA, ouA, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deC, peA, ouA, ocDef, ocDef ) ) ); - + CompleteDataSetRegistration registration = registrationService.getCompleteDataSetRegistration( dsA, peA, ouA, ocDef ); - + assertNotNull( registration ); assertEquals( dsA, registration.getDataSet() ); assertEquals( peA, registration.getPeriod() ); @@ -267,24 +263,24 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetACode.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); - + assertNotNull( summary ); assertNotNull( summary.getImportCount() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 3, dataValues.size() ); assertTrue( dataValues.contains( new DataValue( deA, peA, ouA, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deB, peA, ouA, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deC, peA, ouA, ocDef, ocDef ) ) ); - + CompleteDataSetRegistration registration = registrationService.getCompleteDataSetRegistration( dsA, peA, ouA, ocDef ); - + assertNotNull( registration ); assertEquals( dsA, registration.getDataSet() ); assertEquals( peA, registration.getPeriod() ); @@ -297,7 +293,7 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetB.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); @@ -306,7 +302,7 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 0, summary.getImportCount().getIgnored() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + assertImportDataValues( summary ); } @@ -315,9 +311,13 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetBcode.xml" ).getInputStream(); - - ImportOptions options = new ImportOptions( CODE, CODE, CODE ); - ImportSummary summary = dataValueSetService.saveDataValueSet( in, options ); + + ImportOptions importOptions = new ImportOptions() + .setIdScheme( "CODE" ) + .setDataElementIdScheme( "CODE" ) + .setOrgUnitIdScheme( "CODE" ); + + ImportSummary summary = dataValueSetService.saveDataValueSet( in, importOptions ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); assertEquals( 12, summary.getImportCount().getImported() ); @@ -325,7 +325,7 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 0, summary.getImportCount().getIgnored() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + assertImportDataValues( summary ); } @@ -334,9 +334,14 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetBcode.xml" ).getInputStream(); - - ImportOptions options = new ImportOptions( CODE, CODE, CODE ).setPreheatCache( false ); - ImportSummary summary = dataValueSetService.saveDataValueSet( in, options ); + + ImportOptions importOptions = new ImportOptions() + .setPreheatCache( false ) + .setIdScheme( "CODE" ) + .setDataElementIdScheme( "CODE" ) + .setOrgUnitIdScheme( "CODE" ); + + ImportSummary summary = dataValueSetService.saveDataValueSet( in, importOptions ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); assertEquals( 12, summary.getImportCount().getImported() ); @@ -344,7 +349,7 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 0, summary.getImportCount().getIgnored() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + assertImportDataValues( summary ); } @@ -353,7 +358,7 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetB.csv" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSetCsv( in, null, null ); assertEquals( summary.getConflicts().toString(), 1, summary.getConflicts().size() ); // Header row @@ -362,38 +367,47 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 1, summary.getImportCount().getIgnored() ); // Header row assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + assertImportDataValues( summary ); } - + @Test public void testImportDataValuesXmlDryRun() throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetB.xml" ).getInputStream(); - - ImportOptions options = new ImportOptions( UID, UID, UID ).setDryRun( true ); - - ImportSummary summary = dataValueSetService.saveDataValueSet( in, options ); + + ImportOptions importOptions = new ImportOptions() + .setDryRun( true ) + .setIdScheme( "UID" ) + .setDataElementIdScheme( "UID" ) + .setOrgUnitIdScheme( "UID" ); + + ImportSummary summary = dataValueSetService.saveDataValueSet( in, importOptions ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 0, dataValues.size() ); } - + @Test public void testImportDataValuesXmlUpdatesOnly() throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetB.xml" ).getInputStream(); - - ImportOptions options = new ImportOptions( UID, UID, UID ).setImportStrategy( ImportStrategy.UPDATES ); - - ImportSummary summary = dataValueSetService.saveDataValueSet( in, options ); + + ImportOptions importOptions = new ImportOptions().setImportStrategy( ImportStrategy.UPDATES ); + IdSchemes idSchemes = new IdSchemes(); + idSchemes.setIdScheme( "UID" ); + idSchemes.setDataElementIdScheme( "UID" ); + idSchemes.setOrgUnitIdScheme( "UID" ); + importOptions.setIdSchemes( idSchemes ); + + ImportSummary summary = dataValueSetService.saveDataValueSet( in, importOptions ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); assertEquals( 0, summary.getImportCount().getImported() ); @@ -401,9 +415,9 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 12, summary.getImportCount().getIgnored() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 0, dataValues.size() ); } @@ -420,31 +434,31 @@ assertEquals( 0, summary.getImportCount().getDeleted() ); assertEquals( 0, summary.getImportCount().getIgnored() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 3, dataValues.size() ); } - + @Test public void testImportDataValuesWithAttributeOptionCombo() throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetD.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); assertEquals( summary.getConflicts().toString(), 0, summary.getConflicts().size() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 3, dataValues.size() ); assertTrue( dataValues.contains( new DataValue( deA, peA, ouA, ocDef, ocA ) ) ); assertTrue( dataValues.contains( new DataValue( deB, peA, ouA, ocDef, ocA ) ) ); - assertTrue( dataValues.contains( new DataValue( deC, peA, ouA, ocDef, ocA ) ) ); + assertTrue( dataValues.contains( new DataValue( deC, peA, ouA, ocDef, ocA ) ) ); } @Test @@ -452,14 +466,14 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetE.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); assertEquals( summary.getConflicts().toString(), 2, summary.getConflicts().size() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 1, dataValues.size() ); assertTrue( dataValues.contains( new DataValue( deA, peA, ouA, ocDef, ocA ) ) ); @@ -470,16 +484,16 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetF.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); assertEquals( 0, summary.getImportCount().getImported() ); assertEquals( ImportStatus.ERROR, summary.getStatus() ); - + Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); - assertEquals( 0, dataValues.size() ); + assertEquals( 0, dataValues.size() ); } @Test @@ -487,9 +501,9 @@ throws Exception { in = new ClassPathResource( "datavalueset/dataValueSetG.xml" ).getInputStream(); - + ImportSummary summary = dataValueSetService.saveDataValueSet( in ); - + assertEquals( summary.getConflicts().toString(), 2, summary.getConflicts().size() ); assertEquals( 1, summary.getImportCount().getImported() ); assertEquals( 0, summary.getImportCount().getUpdated() ); @@ -498,9 +512,9 @@ assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); - assertEquals( 1, dataValues.size() ); + assertEquals( 1, dataValues.size() ); } @Test @@ -623,7 +637,7 @@ assertEquals( 2, summary.getImportCount().getImported() ); assertEquals( ImportStatus.SUCCESS, summary.getStatus() ); } - + // ------------------------------------------------------------------------- // Supportive methods // ------------------------------------------------------------------------- @@ -634,7 +648,7 @@ assertNotNull( summary.getImportCount() ); Collection dataValues = mockDataValueBatchHandler.getInserts(); - + assertNotNull( dataValues ); assertEquals( 12, dataValues.size() ); assertTrue( dataValues.contains( new DataValue( deA, peA, ouA, ocDef, ocDef ) ) ); @@ -648,6 +662,6 @@ assertTrue( dataValues.contains( new DataValue( deC, peA, ouA, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deC, peA, ouB, ocDef, ocDef ) ) ); assertTrue( dataValues.contains( new DataValue( deC, peB, ouA, ocDef, ocDef ) ) ); - assertTrue( dataValues.contains( new DataValue( deC, peB, ouB, ocDef, ocDef ) ) ); + assertTrue( dataValues.contains( new DataValue( deC, peB, ouB, ocDef, ocDef ) ) ); } } === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-11-02 18:56:53 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-12-03 07:07:47 +0000 @@ -28,10 +28,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; - +import com.opensymphony.xwork2.Action; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.IdentifiableProperty; @@ -48,7 +45,9 @@ import org.hisp.dhis.user.CurrentUserService; import org.springframework.beans.factory.annotation.Autowired; -import com.opensymphony.xwork2.Action; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; /** * @author Lars Helge Overland @@ -60,7 +59,7 @@ @Autowired private DataValueSetService dataValueSetService; - + @Autowired private AdxDataService adxDataService; @@ -97,7 +96,7 @@ { this.strategy = ImportStrategy.valueOf( stgy ); } - + private IdentifiableProperty idScheme; public void setIdScheme( IdentifiableProperty idScheme ) @@ -134,7 +133,7 @@ } private boolean preheatCache = true; - + public void setPreheatCache( boolean preheatCache ) { this.preheatCache = preheatCache; @@ -158,10 +157,14 @@ in = StreamUtils.wrapAndCheckCompressionFormat( in ); - ImportOptions options = new ImportOptions(). - setIdScheme( idScheme ).setDataElementIdScheme( dataElementIdScheme ).setOrgUnitIdScheme( orgUnitIdScheme ). - setDryRun( dryRun ).setPreheatCache( preheatCache ).setStrategy( strategy ).setSkipExistingCheck( skipExistingCheck ); - + ImportOptions options = new ImportOptions().setDryRun( dryRun ) + .setPreheatCache( preheatCache ).setStrategy( strategy ).setSkipExistingCheck( skipExistingCheck ); + + options.getIdSchemes() + .setIdScheme( idScheme.toString() ) + .setDataElementIdScheme( dataElementIdScheme.toString() ) + .setOrgUnitIdScheme( orgUnitIdScheme.toString() ); + log.info( options ); scheduler.executeTask( new ImportDataValueTask( dataValueSetService, adxDataService, in, options, taskId, importFormat ) ); === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/event/ImportEventAction.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/event/ImportEventAction.java 2015-11-02 03:24:16 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/event/ImportEventAction.java 2015-12-03 07:07:47 +0000 @@ -136,7 +136,7 @@ ImportOptions importOptions = new ImportOptions(); importOptions.setDryRun( dryRun ); - importOptions.setOrgUnitIdScheme( orgUnitIdScheme ); + importOptions.getIdSchemes().setOrgUnitIdScheme( orgUnitIdScheme.toString() ); if ( FORMAT_CSV.equals( payloadFormat ) ) {