=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageBoolAggregator.java 2012-01-09 20:03:56 +0000 @@ -88,7 +88,7 @@ final Map entries = getAggregate( crossTabValues, period.getStartDate(), period.getEndDate(), period.getStartDate(), period.getEndDate(), unitLevel ); // - final Map values = new HashMap( entries.size() ); // + final Map values = new HashMap(); // double average = 0.0; === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/AverageIntSingleValueAggregator.java 2012-01-09 20:03:56 +0000 @@ -87,7 +87,7 @@ final Collection crossTabValues = crossTabService.getCrossTabDataValues( operands, aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key ); - final Map values = new HashMap( crossTabValues.size() * operands.size() ); // + final Map values = new HashMap(); // for ( final CrossTabDataValue crossTabValue : crossTabValues ) { === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumBoolAggregator.java 2012-01-09 20:03:56 +0000 @@ -83,7 +83,7 @@ final Collection crossTabValues = crossTabService.getCrossTabDataValues( operands, aggregationCache.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ), organisationUnits, key ); - final Map values = new HashMap( crossTabValues.size() * operands.size() ); // + final Map values = new HashMap(); // for ( final CrossTabDataValue crossTabValue : crossTabValues ) { === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/aggregation/dataelement/SumIntAggregator.java 2012-01-09 20:03:56 +0000 @@ -87,7 +87,7 @@ final Collection crossTabValues = crossTabService.getCrossTabDataValues( operands, aggregationCache.getPeriodsBetweenDates( period.getStartDate(), period.getEndDate() ), organisationUnits, key ); - final Map values = new HashMap( crossTabValues.size() * operands.size() ); // + final Map values = new HashMap(); // for ( final CrossTabDataValue crossTabValue : crossTabValues ) { === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-01-09 20:03:56 +0000 @@ -131,6 +131,8 @@ final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ). setTableName( CrossTabStore.CROSSTAB_TABLE_PREFIX + key ).init(); + int rows = 0; + for ( final Integer periodId : periodIds ) { for ( final Integer sourceId : organisationUnitIds ) @@ -177,6 +179,8 @@ if ( hasValues ) { + rows++; + batchHandler.addObject( valueList ); } } @@ -186,6 +190,8 @@ statementManager.destroy(); + log.info( "Crosstab table columns: " + ( operands.size() + 2 ) + ", rows: " + rows ); + return null; } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/dataelement/DefaultDataElementDataMart.java 2012-01-09 20:03:56 +0000 @@ -53,6 +53,7 @@ import org.hisp.dhis.organisationunit.OrganisationUnitGroup; import org.hisp.dhis.organisationunit.OrganisationUnitHierarchy; import org.hisp.dhis.period.Period; +import org.hisp.dhis.system.util.SystemUtils; import org.springframework.scheduling.annotation.Async; /** @@ -210,7 +211,7 @@ } } - log.debug( "Exported data values for period: " + period ); + log.debug( "Exported data values for period: " + period + ", " + SystemUtils.getMemoryString() ); } batchHandler.flush(); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-01-09 20:03:56 +0000 @@ -186,7 +186,7 @@ { final int cpuCores = SystemUtils.getCpuCores(); - Clock clock = new Clock().startClock().logTime( "Data mart export process started, number of CPU cores: " + cpuCores ); + Clock clock = new Clock().startClock().logTime( "Data mart export process started, number of CPU cores: " + cpuCores + ", " + SystemUtils.getMemoryString() ); // --------------------------------------------------------------------- // Replace null with empty collection @@ -262,7 +262,7 @@ indicatorOperands.retainAll( allOperands ); - clock.logTime( "Number of operands with data: " + allOperands.size() ); + clock.logTime( "Number of operands with data: " + allOperands.size() + ", " + SystemUtils.getMemoryString() ); // --------------------------------------------------------------------- // Create crosstabtable @@ -286,7 +286,7 @@ ConcurrentUtils.waitForCompletion( crossTabFutures ); - clock.logTime( "Populated crosstab table" ); + clock.logTime( "Populated crosstab table, " + SystemUtils.getMemoryString() ); final boolean isIndicators = indicators != null && indicators.size() > 0; @@ -296,7 +296,7 @@ crossTabService.createAggregatedDataCache( indicatorOperands, key ); - clock.logTime( "Created aggregated data cache" ); + clock.logTime( "Created aggregated data cache, number of indicator operands: " + indicatorOperands.size() + ", operands with data: " + allOperands.size() ); // --------------------------------------------------------------------- // 2. Drop potential indexes @@ -343,7 +343,7 @@ ConcurrentUtils.waitForCompletion( futures ); - clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() ); + clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); } // --------------------------------------------------------------------- @@ -379,7 +379,7 @@ ConcurrentUtils.waitForCompletion( futures ); - clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() ); + clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); } // --------------------------------------------------------------------- @@ -388,7 +388,7 @@ crossTabService.dropAggregatedDataCache( key ); - clock.logTime( "Dropped aggregated data cache" ); + clock.logTime( "Dropped aggregated data cache, " + SystemUtils.getMemoryString() ); // --------------------------------------------------------------------- // 8. Create potential indexes @@ -466,7 +466,7 @@ ConcurrentUtils.waitForCompletion( futures ); - clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() ); + clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); } // --------------------------------------------------------------------- @@ -502,7 +502,7 @@ ConcurrentUtils.waitForCompletion( futures ); - clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() ); + clock.logTime( "Exported values for indicators (" + indicators.size() + "), pages: " + organisationUnitPages.size() + ", " + SystemUtils.getMemoryString() ); } // --------------------------------------------------------------------- @@ -511,7 +511,7 @@ crossTabService.dropAggregatedOrgUnitDataCache( key ); - clock.logTime( "Dropped aggregated org unit data cache" ); + clock.logTime( "Dropped aggregated org unit data cache, " + SystemUtils.getMemoryString() ); // --------------------------------------------------------------------- // 8. Create potential indexes @@ -524,8 +524,7 @@ clock.logTime( "Created org unit indexes" ); } - clock.logTime( "Aggregated organisation unit data export done" ); - + clock.logTime( "Aggregated organisation unit data export done" ); } // --------------------------------------------------------------------- === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/SystemUtils.java 2012-01-09 20:03:56 +0000 @@ -33,6 +33,8 @@ */ public class SystemUtils { + private static final int FACTOR_MB = 1024 * 1024; + public static boolean isRunningForUse() { try @@ -51,4 +53,11 @@ { return Runtime.getRuntime().availableProcessors(); } + + public static String getMemoryString() + { + return "Mem used: " + ( Runtime.getRuntime().totalMemory() / FACTOR_MB ) + + ", max: " + ( Runtime.getRuntime().maxMemory() / FACTOR_MB ) + + ", free: " + ( Runtime.getRuntime().freeMemory() / FACTOR_MB ); + } }