=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultImportService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultImportService.java 2013-10-14 14:38:11 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultImportService.java 2013-12-11 12:28:53 +0000 @@ -33,6 +33,8 @@ import org.hibernate.SessionFactory; import org.hisp.dhis.cache.HibernateCacheManager; import org.hisp.dhis.common.IdentifiableObject; +import org.hisp.dhis.dxf2.timer.SystemNanoTimer; +import org.hisp.dhis.dxf2.timer.Timer; import org.hisp.dhis.scheduling.TaskId; import org.hisp.dhis.system.notification.NotificationLevel; import org.hisp.dhis.system.notification.Notifier; @@ -41,7 +43,6 @@ import org.hisp.dhis.user.UserService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.Assert; import org.springframework.util.StringUtils; import java.util.ArrayList; @@ -67,7 +68,7 @@ // Dependencies //------------------------------------------------------------------------------------------------------- - @Autowired( required = false ) + @Autowired(required = false) private Set importerClasses = new HashSet(); @Autowired @@ -116,16 +117,15 @@ log.info( "User '" + username + "' started import at " + new Date() ); notifier.clear( taskId ).notify( taskId, "Importing meta-data" ); + Timer timer = new SystemNanoTimer(); + timer.start(); ImportSummary importSummary = new ImportSummary(); + objectBridge.setWriteEnabled( !importOptions.isDryRun() ); + objectBridge.setPreheatCache( importOptions.isPreheatCache() ); objectBridge.init(); - if ( importOptions.isDryRun() ) - { - objectBridge.setWriteEnabled( false ); - } - for ( Map.Entry, String> entry : ExchangeClasses.getImportMap().entrySet() ) { Object value = ReflectionUtils.invokeGetterMethod( entry.getValue(), metaData ); @@ -177,14 +177,16 @@ cacheManager.clearCache(); objectBridge.destroy(); + timer.stop(); + if ( taskId != null ) { - notifier.notify( taskId, NotificationLevel.INFO, "Import done", true ). + notifier.notify( taskId, NotificationLevel.INFO, "Import done. Completed in " + timer.toString() + ".", true ). addTaskSummary( taskId, importSummary ); } else { - log.info( "Import done." ); + log.info( "Import done. Completed in " + timer.toString() + "." ); } return importSummary; === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultObjectBridge.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultObjectBridge.java 2013-10-14 14:38:11 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/DefaultObjectBridge.java 2013-12-11 12:28:53 +0000 @@ -32,6 +32,8 @@ import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.dxf2.timer.SystemNanoTimer; +import org.hisp.dhis.dxf2.timer.Timer; import org.hisp.dhis.period.PeriodStore; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.user.User; @@ -41,7 +43,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -89,6 +90,8 @@ private boolean writeEnabled = true; + private boolean preheatCache = true; + //------------------------------------------------------------------------------------------------------- // Build maps //------------------------------------------------------------------------------------------------------- @@ -107,7 +110,9 @@ @Override public void init() { - log.info( "Started updating lookup maps at " + new Date() ); + log.info( "Building object-bridge maps (preheatCache: " + preheatCache + ")." ); + Timer timer = new SystemNanoTimer(); + timer.start(); masterMap = new HashMap, Set>(); periodTypeMap = new HashMap(); @@ -127,7 +132,8 @@ populateIdentifiableObjectMap( type, IdentifiableObject.IdentifiableProperty.NAME ); } - log.info( "Finished updating lookup maps at " + new Date() ); + timer.stop(); + log.info( "Building object-bridge maps took " + timer.toString() + "." ); } @Override @@ -140,6 +146,7 @@ periodTypeMap = null; writeEnabled = true; + preheatCache = true; } //------------------------------------------------------------------------------------------------------- @@ -151,7 +158,7 @@ { Set map = new HashSet(); - if ( IdentifiableObject.class.isAssignableFrom( clazz ) ) + if ( preheatCache && IdentifiableObject.class.isAssignableFrom( clazz ) ) { map = new HashSet( manager.getAll( (Class) clazz ) ); } @@ -164,12 +171,12 @@ { Map map = new HashMap(); - if ( IdentifiableObject.class.isAssignableFrom( clazz ) ) + if ( preheatCache && IdentifiableObject.class.isAssignableFrom( clazz ) ) { map = (Map) manager.getIdMap( (Class) clazz, property ); } - if ( map != null ) + if ( !preheatCache || map != null ) { if ( property == IdentifiableObject.IdentifiableProperty.UID ) { @@ -181,34 +188,41 @@ } else if ( property == IdentifiableObject.IdentifiableProperty.NAME ) { - try - { - IdentifiableObject identifiableObject = (IdentifiableObject) clazz.newInstance(); - - if ( identifiableObject.haveUniqueNames() ) - { - nameMap.put( (Class) clazz, map ); - } - else - { - // add an empty map here, since we could still have some auto-generated properties - nameMap.put( (Class) clazz, new HashMap() ); - - // find all auto-generated props and add them - for ( Map.Entry entry : map.entrySet() ) - { - if ( entry.getValue().isAutoGenerated() ) + if ( !preheatCache ) + { + nameMap.put( (Class) clazz, map ); + } + else + { + try + { + IdentifiableObject identifiableObject = (IdentifiableObject) clazz.newInstance(); + + if ( identifiableObject.haveUniqueNames() ) + { + nameMap.put( (Class) clazz, map ); + } + else + { + // add an empty map here, since we could still have some auto-generated properties + nameMap.put( (Class) clazz, new HashMap() ); + + // find all auto-generated props and add them + for ( Map.Entry entry : map.entrySet() ) { - nameMap.get( clazz ).put( entry.getKey(), entry.getValue() ); + if ( entry.getValue().isAutoGenerated() ) + { + nameMap.get( clazz ).put( entry.getKey(), entry.getValue() ); + } } } } - } - catch ( InstantiationException ignored ) - { - } - catch ( IllegalAccessException ignored ) - { + catch ( InstantiationException ignored ) + { + } + catch ( IllegalAccessException ignored ) + { + } } } } @@ -349,6 +363,18 @@ return writeEnabled; } + @Override + public void setPreheatCache( boolean enabled ) + { + this.preheatCache = enabled; + } + + @Override + public boolean isPreheatCache() + { + return preheatCache; + } + //------------------------------------------------------------------------------------------------------- // Internal Methods //------------------------------------------------------------------------------------------------------- @@ -468,37 +494,55 @@ private IdentifiableObject getUidMatch( IdentifiableObject identifiableObject ) { Map map = uidMap.get( identifiableObject.getClass() ); + IdentifiableObject entity = null; if ( map != null ) { - return map.get( identifiableObject.getUid() ); - } - - return null; + entity = map.get( identifiableObject.getUid() ); + } + + if ( !preheatCache && entity == null ) + { + entity = manager.get( identifiableObject.getClass(), identifiableObject.getUid() ); + } + + return entity; } private IdentifiableObject getCodeMatch( IdentifiableObject identifiableObject ) { Map map = codeMap.get( identifiableObject.getClass() ); + IdentifiableObject entity = null; if ( map != null ) { - return map.get( identifiableObject.getCode() ); - } - - return null; + entity = map.get( identifiableObject.getCode() ); + } + + if ( !preheatCache && entity == null ) + { + entity = manager.getByCode( identifiableObject.getClass(), identifiableObject.getCode() ); + } + + return entity; } private IdentifiableObject getNameMatch( IdentifiableObject identifiableObject ) { Map map = nameMap.get( identifiableObject.getClass() ); + IdentifiableObject entity = null; if ( map != null ) { - return map.get( identifiableObject.getName() ); - } - - return null; + entity = map.get( identifiableObject.getName() ); + } + + if ( !preheatCache && identifiableObject.haveUniqueNames() && entity == null ) + { + entity = manager.getByName( identifiableObject.getClass(), identifiableObject.getName() ); + } + + return entity; } private boolean _typeSupported( Class clazz ) === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ImportOptions.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ImportOptions.java 2013-09-04 11:58:22 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ImportOptions.java 2013-12-11 12:28:53 +0000 @@ -42,13 +42,15 @@ private boolean dryRun; + private boolean preheatCache = true; + private boolean async; private ImportStrategy importStrategy; private boolean skipExistingCheck; - private static final ImportOptions DEFAULT_OPTIONS = new ImportOptions( IdentifiableProperty.UID, IdentifiableProperty.UID, false, ImportStrategy.NEW_AND_UPDATES, false ); + private static final ImportOptions DEFAULT_OPTIONS = new ImportOptions( IdentifiableProperty.UID, IdentifiableProperty.UID, false, true, ImportStrategy.NEW_AND_UPDATES, false ); public static ImportOptions getDefaultImportOptions() { @@ -64,10 +66,11 @@ this.importStrategy = importStrategy; } - public ImportOptions( IdentifiableProperty dataElementIdScheme, IdentifiableProperty orgUnitIdScheme, boolean dryRun, ImportStrategy importStrategy, boolean skipExistingCheck ) + public ImportOptions( IdentifiableProperty dataElementIdScheme, IdentifiableProperty orgUnitIdScheme, boolean dryRun, boolean preheatCache, ImportStrategy importStrategy, boolean skipExistingCheck ) { this.dataElementIdScheme = dataElementIdScheme; this.orgUnitIdScheme = orgUnitIdScheme; + this.preheatCache = preheatCache; this.dryRun = dryRun; this.importStrategy = importStrategy; this.skipExistingCheck = skipExistingCheck; @@ -92,6 +95,11 @@ return dryRun; } + public boolean isPreheatCache() + { + return preheatCache; + } + public ImportStrategy getImportStrategy() { return importStrategy != null ? importStrategy : ImportStrategy.NEW_AND_UPDATES; @@ -121,6 +129,11 @@ this.dryRun = dryRun; } + public void setPreheatCache( boolean preheatCache ) + { + this.preheatCache = preheatCache; + } + public boolean isAsync() { return async; === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ObjectBridge.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ObjectBridge.java 2013-08-23 16:05:01 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/ObjectBridge.java 2013-12-11 12:28:53 +0000 @@ -97,4 +97,19 @@ * @return {@code boolean} indicating status of {@code writeEnabled} */ boolean isWriteEnabled(); + + /** + * Enable or disable preheating the internal cache. This should be left on for most cases, + * but for very small imports (1-10 objects) turning this off will generally speed up import by a factor of 100. + * + * @param enabled {@code boolean} turning preheating on or off + */ + void setPreheatCache( boolean enabled ); + + /** + * Is preheat cache enabled? + * + * @return {@code boolean} indicating status of {@code preheatCache} + */ + boolean isPreheatCache(); } === added directory 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer' === added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/SystemNanoTimer.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/SystemNanoTimer.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/SystemNanoTimer.java 2013-12-11 12:28:53 +0000 @@ -0,0 +1,66 @@ +package org.hisp.dhis.dxf2.timer; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.concurrent.TimeUnit; + +/** + * @author Morten Olav Hansen + */ +public class SystemNanoTimer implements Timer +{ + private long _start = 0; + + private long _end = 0; + + @Override + public void start() + { + _start = System.nanoTime(); + } + + @Override + public void stop() + { + _end = System.nanoTime(); + } + + @Override + public Long getDuration() + { + return _end - _start; + } + + @Override + public String toString() + { + double seconds = TimeUnit.MILLISECONDS.convert( getDuration(), TimeUnit.NANOSECONDS ) / 1000.0f; + return String.format( "%.2f seconds", seconds ); + } +} === added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/Timer.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/Timer.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/timer/Timer.java 2013-12-11 12:28:53 +0000 @@ -0,0 +1,43 @@ +package org.hisp.dhis.dxf2.timer; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Simple interface that captures time, and pretty prints it back to you + * + * @author Morten Olav Hansen + */ +public interface Timer +{ + void start(); + + void stop(); + + T getDuration(); +} === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java 2013-08-23 16:05:01 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceTest.java 2013-12-11 12:28:53 +0000 @@ -179,7 +179,7 @@ public void testImportDataValuesXmlWithCode() throws Exception { - ImportOptions options = new ImportOptions( CODE, CODE, false, NEW_AND_UPDATES, false ); + ImportOptions options = new ImportOptions( CODE, CODE, false, true, NEW_AND_UPDATES, false ); ImportSummary summary = dataValueSetService.saveDataValueSet( new ClassPathResource( "datavalueset/dataValueSetBcode.xml" ).getInputStream(), options ); assertImportDataValues( summary ); @@ -222,7 +222,7 @@ public void testImportDataValuesXmlDryRun() throws Exception { - ImportOptions options = new ImportOptions( UID, UID, true, NEW_AND_UPDATES, false ); + ImportOptions options = new ImportOptions( UID, UID, true, true, NEW_AND_UPDATES, false ); dataValueSetService.saveDataValueSet( new ClassPathResource( "datavalueset/dataValueSetB.xml" ).getInputStream(), options ); @@ -236,7 +236,7 @@ public void testImportDataValuesXmlUpdatesOnly() throws Exception { - ImportOptions options = new ImportOptions( UID, UID, false, UPDATES, false ); + ImportOptions options = new ImportOptions( UID, UID, false, true, UPDATES, false ); dataValueSetService.saveDataValueSet( new ClassPathResource( "datavalueset/dataValueSetB.xml" ).getInputStream(), options ); === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/PDFFormController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/PDFFormController.java 2013-08-23 16:00:30 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/api/controller/PDFFormController.java 2013-12-11 12:28:53 +0000 @@ -164,7 +164,7 @@ boolean dryRun = false; boolean skipExistingCheck = false; - ImportOptions options = new ImportOptions( dataElementIdScheme, orgUnitIdScheme, dryRun, strategy, + ImportOptions options = new ImportOptions( dataElementIdScheme, orgUnitIdScheme, dryRun, true, strategy, skipExistingCheck ); log.info( options ); === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2013-08-28 07:27:01 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2013-12-11 12:28:53 +0000 @@ -145,7 +145,7 @@ in = StreamUtils.wrapAndCheckCompressionFormat( in ); - ImportOptions options = new ImportOptions( dataElementIdScheme, orgUnitIdScheme, dryRun, strategy, skipExistingCheck ); + ImportOptions options = new ImportOptions( dataElementIdScheme, orgUnitIdScheme, dryRun, true, strategy, skipExistingCheck ); log.info( options );