=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-06-06 05:46:14 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-10-22 08:41:00 +0000 @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.datamart.CrossTabDataValue; @@ -50,6 +51,8 @@ * @return the DataElementOperands with data. */ Set getOperandsWithData( Set operands ); + + String createCrossTabTable( List operands ); /** * Creates and populates the crosstab table. Operands without data will be @@ -60,8 +63,8 @@ * @param organisationUnitIds the collection of OrganisationUnit identifiers. * @return a List of random keys for each generated crosstab table. */ - String populateCrossTabTable( List operands, - Collection periodIds, Collection organisationUnitIds ); + Future populateCrossTabTable( List operands, + Collection periodIds, Collection organisationUnitIds, String key ); /** * Drops the crosstab table. === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-06-04 16:56:41 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-10-22 08:41:00 +0000 @@ -32,9 +32,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import org.amplecode.quick.BatchHandler; import org.amplecode.quick.BatchHandlerFactory; +import org.amplecode.quick.StatementManager; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,6 +46,7 @@ import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler; +import org.springframework.scheduling.annotation.Async; /** * @author Lars Helge Overland @@ -86,6 +89,13 @@ { this.dataValueService = dataValueService; } + + private StatementManager statementManager; + + public void setStatementManager( StatementManager statementManager ) + { + this.statementManager = statementManager; + } // ------------------------------------------------------------------------- // CrossTabService implementation @@ -96,62 +106,67 @@ return dataValueService.getOperandsWithDataValues( operands ); } - public String populateCrossTabTable( List operands, - Collection periodIds, Collection organisationUnitIds ) + public String createCrossTabTable( List operands ) { final String key = RandomStringUtils.randomAlphanumeric( 8 ); - if ( validate( operands, periodIds, organisationUnitIds ) ) + crossTabStore.dropCrossTabTable( key ); + crossTabStore.createCrossTabTable( operands, key ); + + return key; + } + + @Async + public Future populateCrossTabTable( List operands, + Collection periodIds, Collection organisationUnitIds, String key ) + { + statementManager.initialise(); + + final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ). + setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init(); + + for ( final Integer periodId : periodIds ) { - crossTabStore.dropCrossTabTable( key ); - crossTabStore.createCrossTabTable( operands, key ); - - final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ). - setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init(); - - for ( final Integer periodId : periodIds ) + for ( final Integer sourceId : organisationUnitIds ) { - for ( final Integer sourceId : organisationUnitIds ) - { - final Map map = aggregatedDataValueService.getDataValueMap( periodId, sourceId ); - - final List valueList = new ArrayList( operands.size() + 2 ); - - valueList.add( String.valueOf( periodId ) ); - valueList.add( String.valueOf( sourceId ) ); - - boolean hasValues = false; - - for ( DataElementOperand operand : operands ) - { - String value = map.get( operand ); - - if ( value != null && value.length() > MAX_LENGTH ) - { - log.warn( "Value ignored, too long: '" + value + "'" ); - value = null; - } - - if ( value != null ) - { - hasValues = true; - } - - valueList.add( value ); - } - - if ( hasValues ) - { - batchHandler.addObject( valueList ); - } + final Map map = aggregatedDataValueService.getDataValueMap( periodId, sourceId ); + + final List valueList = new ArrayList( operands.size() + 2 ); + + valueList.add( String.valueOf( periodId ) ); + valueList.add( String.valueOf( sourceId ) ); + + boolean hasValues = false; + + for ( DataElementOperand operand : operands ) + { + String value = map.get( operand ); + + if ( value != null && value.length() > MAX_LENGTH ) + { + log.warn( "Value ignored, too long: '" + value + "'" ); + value = null; + } + + if ( value != null ) + { + hasValues = true; + } + + valueList.add( value ); + } + + if ( hasValues ) + { + batchHandler.addObject( valueList ); } } - - batchHandler.flush(); - - return key; } - + + batchHandler.flush(); + + statementManager.destroy(); + return null; } @@ -187,36 +202,4 @@ { return crossTabStore.getAggregatedDataCacheValue( operands, periodId, sourceId, key ); } - - // ------------------------------------------------------------------------- - // Supportive methods - // ------------------------------------------------------------------------- - - /** - * Validates whether the given collections of identifiers are not null and - * of size greater than 0. - */ - private boolean validate( Collection operands, Collection periodIds, - Collection unitIds ) - { - if ( operands == null || operands.size() == 0 ) - { - log.warn( "No operands selected for crosstab table" ); - return false; - } - - if ( periodIds == null || periodIds.size() == 0 ) - { - log.warn( "No periods selected for crosstab table" ); - return false; - } - - if ( unitIds == null || unitIds.size() == 0 ) - { - log.warn( "No organisation units selected for crosstab table" ); - return false; - } - - return true; - } } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-10-19 14:01:02 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-10-22 08:41:00 +0000 @@ -173,7 +173,7 @@ try { - final Map valueMap = new HashMap(); + final Map valueMap = new HashMap( operands.size() ); final ResultSet resultSet = holder.getStatement().executeQuery( sql ); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-10-08 13:31:38 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-10-22 08:41:00 +0000 @@ -223,10 +223,21 @@ state.setMessage( "crosstabulating_data" ); + Collection intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) ); Collection childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( organisationUnitIds ); - Collection intersectingPeriodIds = ConversionUtils.getIdentifiers( Period.class, periodService.getIntersectionPeriods( periods ) ); - - String key = crossTabService.populateCrossTabTable( new ArrayList( allOperands ), intersectingPeriodIds, childrenIds ); + List> childrenPages = new PaginatedList( childrenIds ).setNumberOfPages( cpuCores ).getPages(); + + List crossTabOperands = new ArrayList( allOperands ); + String key = crossTabService.createCrossTabTable( crossTabOperands ); + + List> crossTabFutures = new ArrayList>(); + + for ( List childrenPage : childrenPages ) + { + crossTabFutures.add( crossTabService.populateCrossTabTable( crossTabOperands, intersectingPeriodIds, childrenPage, key ) ); + } + + ConcurrentUtils.waitForCompletion( crossTabFutures ); clock.logTime( "Populated crosstab table" ); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-10-18 20:48:23 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-10-22 08:41:00 +0000 @@ -53,6 +53,7 @@ + === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-06-04 16:56:41 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-10-22 08:41:00 +0000 @@ -189,8 +189,10 @@ @Test public void testPopulateCrossTabValue() + throws Exception { - String key = crossTabService.populateCrossTabTable( operands, periodIds, organisationUnitIds ); + String key = crossTabService.createCrossTabTable( operands ); + crossTabService.populateCrossTabTable( operands, periodIds, organisationUnitIds, key ).get(); Collection values = crossTabService.getCrossTabDataValues( operands, periodIds, organisationUnitIds, key );