=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-22 16:17:24 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-08-22 17:05:32 +0000 @@ -316,155 +316,81 @@ clock.logTime( "Populated crosstab table, " + SystemUtils.getMemoryString() ); notifier.notify( id, DATAMART, "Exporting data element data" ); + final boolean isDataElements = true; + final boolean isIndicators = indicators != null && indicators.size() > 0; - - // --------------------------------------------------------------------- - // 1. Export data element values - // --------------------------------------------------------------------- - - List> organisationUnitPages = new PaginatedList( organisationUnits ).setNumberOfPages( cpuCores ).getPages(); - - if ( allOperands.size() > 0 ) - { - final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits ); - - List> futures = new ArrayList>(); - - for ( List organisationUnitPage : organisationUnitPages ) - { - futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage, - null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) ); - } - - ConcurrentUtils.waitForCompletion( futures ); - } - - clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); - notifier.notify( id, DATAMART, "Dropping data element index" ); - - // --------------------------------------------------------------------- - // 2. Drop data element index - // --------------------------------------------------------------------- - - dataMartManager.dropDataValueIndex(); - - clock.logTime( "Dropped data element index" ); - notifier.notify( id, DATAMART, "Deleting existing data element data" ); - - // --------------------------------------------------------------------- - // 3. Delete existing aggregated data values - // --------------------------------------------------------------------- - - dataMartManager.deleteAggregatedDataValues( periodIds ); - - clock.logTime( "Deleted existing data element data" ); - notifier.notify( id, DATAMART, "Copying data element data from temporary table" ); - - // --------------------------------------------------------------------- - // 4. Copy aggregated data values from temporary table - // --------------------------------------------------------------------- - - dataMartManager.copyAggregatedDataValuesFromTemp(); - - clock.logTime( "Copied data element data from temporary table" ); - notifier.notify( id, DATAMART, "Creating data element index" ); - - // --------------------------------------------------------------------- - // 5. Create data element index - // --------------------------------------------------------------------- - - dataMartManager.createDataValueIndex(); - - clock.logTime( "Created data element index" ); - - if ( isIndicators ) - { - // ----------------------------------------------------------------- - // 6. Create and populate aggregated data cache - // ----------------------------------------------------------------- - - notifier.notify( id, DATAMART, "Populating aggregated data cache" ); - - crossTabService.createAggregatedDataCache( indicatorOperands, key ); - - List> aggregatedDataCacheFutures = new ArrayList>(); - - for ( List organisationUnitPage : organisationUnitPages ) - { - aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache( - indicatorOperands, periods, organisationUnitPage, key ) ); - } - - ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures ); - - clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() ); - notifier.notify( id, DATAMART, "Exporting indicator data" ); - - // ----------------------------------------------------------------- - // 7. Export indicator values - // ----------------------------------------------------------------- - - List> futures = new ArrayList>(); - - for ( List organisationUnitPage : organisationUnitPages ) - { - futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, - null, indicatorOperands, AggregatedIndicatorValueTempBatchHandler.class, key ) ); - } - - ConcurrentUtils.waitForCompletion( futures ); - - clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); - notifier.notify( id, DATAMART, "Dropping indicator index" ); - - // ----------------------------------------------------------------- - // 8. Drop aggregated data cache and indicator index - // ----------------------------------------------------------------- - - crossTabService.dropAggregatedDataCache( key ); - dataMartManager.dropIndicatorValueIndex(); - - clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() ); - notifier.notify( id, DATAMART, "Deleting existing indicator data" ); - - // ----------------------------------------------------------------- - // 9. Delete existing aggregated indicator values - // ----------------------------------------------------------------- - - dataMartManager.deleteAggregatedIndicatorValues( periodIds ); - - clock.logTime( "Deleted existing indicator data" ); - notifier.notify( id, DATAMART, "Copying indicator data from temporary table" ); - - // ----------------------------------------------------------------- - // 10. Copy aggregated data values from temporary table - // ----------------------------------------------------------------- - - dataMartManager.copyAggregatedIndicatorValuesFromTemp(); - - clock.logTime( "Copied indicator data from temporary table" ); - notifier.notify( id, DATAMART, "Creating indicator index" ); - - // ----------------------------------------------------------------- - // 11. Create indicator index - // ----------------------------------------------------------------- - - dataMartManager.createIndicatorValueIndex(); - - clock.logTime( "Created indicator index" ); - } - - clock.logTime( "Aggregated data export done" ); final int groupLevel = (Integer) systemSettingManager.getSystemSetting( KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL ); final boolean isGroups = organisationUnitGroups != null && organisationUnitGroups.size() > 0 && groupLevel > 0; - - if ( isGroups ) - { - // --------------------------------------------------------------------- - // 1. Export data element values - // --------------------------------------------------------------------- + + List> organisationUnitPages = new PaginatedList( organisationUnits ).setNumberOfPages( cpuCores ).getPages(); + + if ( isDataElements ) + { + // ----------------------------------------------------------------- + // 1. Export data element values + // ----------------------------------------------------------------- + + if ( allOperands.size() > 0 ) + { + final OrganisationUnitHierarchy hierarchy = organisationUnitService.getOrganisationUnitHierarchy().prepareChildren( organisationUnits ); + + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage, + null, hierarchy, AggregatedDataValueTempBatchHandler.class, key ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + } + + clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); + notifier.notify( id, DATAMART, "Dropping data element index" ); + + // ----------------------------------------------------------------- + // 2. Drop data element index + // ----------------------------------------------------------------- + + dataMartManager.dropDataValueIndex(); + + clock.logTime( "Dropped data element index" ); + notifier.notify( id, DATAMART, "Deleting existing data element data" ); + + // ----------------------------------------------------------------- + // 3. Delete existing aggregated data values + // ----------------------------------------------------------------- + + dataMartManager.deleteAggregatedDataValues( periodIds ); + + clock.logTime( "Deleted existing data element data" ); + notifier.notify( id, DATAMART, "Copying data element data from temporary table" ); + + // ----------------------------------------------------------------- + // 4. Copy aggregated data values from temporary table + // ----------------------------------------------------------------- + + dataMartManager.copyAggregatedDataValuesFromTemp(); + + clock.logTime( "Copied data element data from temporary table" ); + notifier.notify( id, DATAMART, "Creating data element index" ); + + // ----------------------------------------------------------------- + // 5. Create data element index + // ----------------------------------------------------------------- + + dataMartManager.createDataValueIndex(); + + clock.logTime( "Created data element index" ); + } + + if ( isGroups && isDataElements ) + { + // ----------------------------------------------------------------- + // 1. Export data element values + // ----------------------------------------------------------------- notifier.notify( id, DATAMART, "Exporting data element org unit data" ); @@ -501,118 +427,194 @@ clock.logTime( "Dropped org unit data element index" ); notifier.notify( id, DATAMART, "Deleting existing org unit data element data" ); - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- // 3. Delete existing aggregated data values - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- dataMartManager.deleteAggregatedOrgUnitDataValues( periodIds ); clock.logTime( "Deleted existing aggregated org unit datavalues" ); notifier.notify( id, DATAMART, "Copying org unit data element data" ); - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- // 4. Copy aggregated org unit data values from temporary table - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- dataMartManager.copyAggregatedOrgUnitDataValuesFromTemp(); clock.logTime( "Copied org unit data element data from temporary table" ); notifier.notify( id, DATAMART, "Creating org unit data element index" ); - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- // 5. Create org unit data element index - // --------------------------------------------------------------------- + // ----------------------------------------------------------------- dataMartManager.createOrgUnitDataValueIndex(); clock.logTime( "Created org unit data element index" ); notifier.notify( id, DATAMART, "Exporting data for org unit indicator data" ); - - if ( isIndicators ) - { - // ----------------------------------------------------------------- - // 6. Create aggregated data cache - // ----------------------------------------------------------------- - - notifier.notify( id, DATAMART, "Populating aggregated data cache" ); - - crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key ); - - List> aggregatedDataCacheFutures = new ArrayList>(); - - for ( List organisationUnitPage : organisationUnitPages ) - { - aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache( - indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key ) ); - } - - ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures ); - - clock.logTime( "Created aggregated org unit data cache" ); - notifier.notify( id, DATAMART, "Exporting org unit indicator data" ); - - // --------------------------------------------------------------------- - // 7. Export indicator values - // --------------------------------------------------------------------- - - List> futures = new ArrayList>(); - - for ( List organisationUnitPage : organisationUnitPages ) - { - futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, - organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueTempBatchHandler.class, key ) ); - } - - ConcurrentUtils.waitForCompletion( futures ); - - clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); - notifier.notify( id, DATAMART, "Dropping org unit indicator index" ); - - // --------------------------------------------------------------------- - // 8. Drop aggregated data cache and indicator index - // --------------------------------------------------------------------- - - crossTabService.dropAggregatedOrgUnitDataCache( key ); - dataMartManager.dropOrgUnitIndicatorValueIndex(); - - clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() ); - notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" ); - - // --------------------------------------------------------------------- - // 9. Delete existing aggregated indicator values - // --------------------------------------------------------------------- - - dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds ); - - clock.logTime( "Deleted existing aggregated org unit indicatorvalues" ); - notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" ); - - // --------------------------------------------------------------------- - // 10. Copy aggregated org unit indicator values from temporary table - // --------------------------------------------------------------------- - - dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp(); - - clock.logTime( "Copied org unit indicator data from temporary table" ); - notifier.notify( id, DATAMART, "Creating org unit indicator indexes" ); - - // --------------------------------------------------------------------- - // 11. Create org unit indicator index - // --------------------------------------------------------------------- - - dataMartManager.createOrgUnitIndicatorValueIndex(); - - clock.logTime( "Created org unit indicator index" ); - } - - clock.logTime( "Aggregated org unit data export done" ); } - // --------------------------------------------------------------------- - // Drop crosstab and temporary tables - // --------------------------------------------------------------------- - crossTabService.dropCrossTabTable( key ); + + if ( isIndicators ) + { + // ----------------------------------------------------------------- + // 1. Create and populate aggregated data cache + // ----------------------------------------------------------------- + + notifier.notify( id, DATAMART, "Populating aggregated data cache" ); + + crossTabService.createAggregatedDataCache( indicatorOperands, key ); + + List> aggregatedDataCacheFutures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + aggregatedDataCacheFutures.add( crossTabService.populateAggregatedDataCache( + indicatorOperands, periods, organisationUnitPage, key ) ); + } + + ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures ); + + clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() ); + notifier.notify( id, DATAMART, "Exporting indicator data" ); + + // ----------------------------------------------------------------- + // 2. Export indicator values + // ----------------------------------------------------------------- + + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, + null, indicatorOperands, AggregatedIndicatorValueTempBatchHandler.class, key ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + + clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); + notifier.notify( id, DATAMART, "Dropping indicator index" ); + + // ----------------------------------------------------------------- + // 3. Drop aggregated data cache and indicator index + // ----------------------------------------------------------------- + + crossTabService.dropAggregatedDataCache( key ); + dataMartManager.dropIndicatorValueIndex(); + + clock.logTime( "Dropped indicator index, " + SystemUtils.getMemoryString() ); + notifier.notify( id, DATAMART, "Deleting existing indicator data" ); + + // ----------------------------------------------------------------- + // 4. Delete existing aggregated indicator values + // ----------------------------------------------------------------- + + dataMartManager.deleteAggregatedIndicatorValues( periodIds ); + + clock.logTime( "Deleted existing indicator data" ); + notifier.notify( id, DATAMART, "Copying indicator data from temporary table" ); + + // ----------------------------------------------------------------- + // 5. Copy aggregated data values from temporary table + // ----------------------------------------------------------------- + + dataMartManager.copyAggregatedIndicatorValuesFromTemp(); + + clock.logTime( "Copied indicator data from temporary table" ); + notifier.notify( id, DATAMART, "Creating indicator index" ); + + // ----------------------------------------------------------------- + // 6. Create indicator index + // ----------------------------------------------------------------- + + dataMartManager.createIndicatorValueIndex(); + + clock.logTime( "Created indicator index" ); + } + + if ( isGroups && isIndicators ) + { + // ----------------------------------------------------------------- + // 1. Create aggregated data cache + // ----------------------------------------------------------------- + + notifier.notify( id, DATAMART, "Populating aggregated data cache" ); + + crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key ); + + List> aggregatedDataCacheFutures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + aggregatedDataCacheFutures.add( crossTabService.populateAggregatedOrgUnitDataCache( + indicatorOperands, periods, organisationUnitPage, organisationUnitGroups, key ) ); + } + + ConcurrentUtils.waitForCompletion( aggregatedDataCacheFutures ); + + clock.logTime( "Created aggregated org unit data cache" ); + notifier.notify( id, DATAMART, "Exporting org unit indicator data" ); + + // ----------------------------------------------------------------- + // 2. Export indicator values + // ----------------------------------------------------------------- + + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, + organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueTempBatchHandler.class, key ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + + clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); + notifier.notify( id, DATAMART, "Dropping org unit indicator index" ); + + // ----------------------------------------------------------------- + // 3. Drop aggregated data cache and indicator index + // ----------------------------------------------------------------- + + crossTabService.dropAggregatedOrgUnitDataCache( key ); + dataMartManager.dropOrgUnitIndicatorValueIndex(); + + clock.logTime( "Dropped org unit indicator index, " + SystemUtils.getMemoryString() ); + notifier.notify( id, DATAMART, "Deleting existing org unit indicator data" ); + + // ----------------------------------------------------------------- + // 4. Delete existing aggregated indicator values + // ----------------------------------------------------------------- + + dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds ); + + clock.logTime( "Deleted existing aggregated org unit indicatorvalues" ); + notifier.notify( id, DATAMART, "Copying org unit indicator data from temporary table" ); + + // ----------------------------------------------------------------- + // 5. Copy aggregated org unit indicator values from temp table + // ----------------------------------------------------------------- + + dataMartManager.copyAggregatedOrgUnitIndicatorValuesFromTemp(); + + clock.logTime( "Copied org unit indicator data from temporary table" ); + notifier.notify( id, DATAMART, "Creating org unit indicator indexes" ); + + // ----------------------------------------------------------------- + // 6. Create org unit indicator index + // ----------------------------------------------------------------- + + dataMartManager.createOrgUnitIndicatorValueIndex(); + + clock.logTime( "Created org unit indicator index" ); + } + + // --------------------------------------------------------------------- + // Drop temporary tables + // --------------------------------------------------------------------- + dataMartManager.dropTempAggregatedTables(); clock.logTime( "Dropped crosstab table" );