=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/Partitions.java 2014-03-24 21:32:04 +0000 @@ -95,13 +95,13 @@ /** * Prunes this instance so that it retains only the partitions included in - * the given list. No operation takes place if the given live is null. + * the given list. No operation takes place if the given list is null or empty. * * @param validPartitions list of valid partitions to retain. */ public Partitions prunePartitions( List validPartitions ) { - if ( validPartitions != null ) + if ( validPartitions != null && !validPartitions.isEmpty() ) { partitions.retainAll( validPartitions ); } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/EventQueryPlanner.java 2014-03-24 22:19:59 +0000 @@ -46,8 +46,8 @@ * Plans the given params and returns a list of params. * * @param params the query params. - * @param validPartitions the list of existing database partition names, only - * required for aggregate queries. */ - List planQuery( EventQueryParams params, List validPartitions ); + List planAggregateQuery( EventQueryParams params ); + + EventQueryParams planEventQuery( EventQueryParams params ); } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-03-24 22:19:59 +0000 @@ -125,14 +125,11 @@ // TODO order event analytics tables on execution date to avoid default // TODO sorting in queries - // TODO parallel processing of queries public Grid getAggregatedEventData( EventQueryParams params ) { queryPlanner.validate( params ); - List validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() ); - Grid grid = new ListGrid(); // --------------------------------------------------------------------- @@ -155,7 +152,7 @@ // Data // --------------------------------------------------------------------- - List queries = queryPlanner.planQuery( params, validPartitions ); + List queries = queryPlanner.planAggregateQuery( params ); for ( EventQueryParams query : queries ) { @@ -222,24 +219,24 @@ Timer t = new Timer().start(); - List queries = queryPlanner.planQuery( params, null ); + params = queryPlanner.planEventQuery( params ); - t.getSplitTime( "Planned query, got: " + queries.size() ); + t.getSplitTime( "Planned query, got partitions: " + params.getPartitions() ); int count = 0; - for ( EventQueryParams query : queries ) + if ( params.getPartitions().hasAny() ) { if ( params.isPaging() ) { - count += analyticsManager.getEventCount( query ); + count += analyticsManager.getEventCount( params ); } - - analyticsManager.getEvents( query, grid ); + + analyticsManager.getEvents( params, grid ); + + t.getTime( "Queried events, got: " + grid.getHeight() ); } - - t.getTime( "Queried events, got: " + grid.getHeight() ); - + // --------------------------------------------------------------------- // Meta-data // --------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventQueryPlanner.java 2014-03-24 22:19:59 +0000 @@ -31,7 +31,6 @@ import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import java.util.ArrayList; -import java.util.Date; import java.util.List; import org.apache.commons.logging.Log; @@ -39,15 +38,16 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.Partitions; import org.hisp.dhis.analytics.QueryPlanner; +import org.hisp.dhis.analytics.event.EventAnalyticsManager; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.EventQueryPlanner; import org.hisp.dhis.analytics.table.PartitionUtils; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.ListMap; import org.hisp.dhis.common.NameableObject; -import org.hisp.dhis.period.Cal; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; -import org.hisp.dhis.program.Program; import org.springframework.beans.factory.annotation.Autowired; /** @@ -60,7 +60,18 @@ @Autowired private QueryPlanner queryPlanner; + + @Autowired + private EventAnalyticsManager analyticsManager; + @Autowired + private OrganisationUnitService organisationUnitService; + + // ------------------------------------------------------------------------- + // EventQueryPlanner implementation + // ------------------------------------------------------------------------- + + @Override public void validate( EventQueryParams params ) throws IllegalQueryException { @@ -107,8 +118,11 @@ } } - public List planQuery( EventQueryParams params, List validPartitions ) + @Override + public List planAggregateQuery( EventQueryParams params ) { + List validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() ); + List queries = new ArrayList(); List groupedByPartition = groupByPartition( params, validPartitions ); @@ -125,63 +139,58 @@ return queries; } - + + @Override + public EventQueryParams planEventQuery( EventQueryParams params ) + { + List validPartitions = analyticsManager.getAnalyticsTables( params.getProgram() ); + + String tableSuffix = "_" + params.getProgram().getUid(); + + if ( params.hasStartEndDate() ) + { + Period queryPeriod = new Period(); + queryPeriod.setStartDate( params.getStartDate() ); + queryPeriod.setEndDate( params.getEndDate() ); + params.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) ); + } + + for ( NameableObject object : params.getOrganisationUnits() ) + { + OrganisationUnit unit = (OrganisationUnit) object; + unit.setLevel( organisationUnitService.getLevelOfOrganisationUnit( unit.getUid() ) ); + } + + //TODO periods, convert to start/end dates + + return params; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + private List groupByPartition( EventQueryParams params, List validPartitions ) { List queries = new ArrayList(); - Program program = params.getProgram(); - - String tableSuffix = "_" + program.getUid(); + String tableSuffix = "_" + params.getProgram().getUid(); if ( params.hasStartEndDate() ) { - if ( params.isAggregate() ) // Multiple partitions/years in one query - { - Period queryPeriod = new Period(); - queryPeriod.setStartDate( params.getStartDate() ); - queryPeriod.setEndDate( params.getEndDate() ); - - EventQueryParams query = params.instance(); - query.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) ); - - if ( query.getPartitions().hasAny() ) - { - queries.add( query ); - } - } - else // Event query - split in one query per partition/year - { - Date startDate = params.getStartDate(); - Date endDate = params.getEndDate(); - - Date currentStartDate = startDate; - Date currentEndDate = endDate; - - while ( true ) - { - if ( PartitionUtils.year( currentStartDate ) < PartitionUtils.year( endDate ) ) // Spans multiple - { - // Set end date to max of current year - - currentEndDate = PartitionUtils.maxOfYear( currentStartDate ); - - queries.add( getQuery( params, currentStartDate, currentEndDate, program ) ); - - // Set start date to start of next year - - currentStartDate = new Cal( ( PartitionUtils.year( currentStartDate ) + 1 ), 1, 1 ).time(); - } - else - { - queries.add( getQuery( params, currentStartDate, endDate, program ) ); - - break; - } - } + Period queryPeriod = new Period(); + queryPeriod.setStartDate( params.getStartDate() ); + queryPeriod.setEndDate( params.getEndDate() ); + + EventQueryParams query = params.instance(); + query.setPartitions( PartitionUtils.getPartitions( queryPeriod, TABLE_PREFIX, tableSuffix, validPartitions ) ); + + if ( query.getPartitions().hasAny() ) + { + queries.add( query ); } } - else + else // Aggregate only { ListMap partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getDimensionOrFilter( PERIOD_DIM_ID ), TABLE_PREFIX, tableSuffix ); @@ -196,17 +205,7 @@ return queries; } - - private EventQueryParams getQuery( EventQueryParams params, Date startDate, Date endDate, Program program ) - { - EventQueryParams query = params.instance(); - query.setStartDate( startDate ); - query.setEndDate( endDate ); - String tableName = TABLE_PREFIX + "_" + PartitionUtils.year( startDate ) + "_" + program.getUid(); - query.setPartitions( new Partitions().add( tableName ) ); - return query; - } - + private static List convert( List params ) { List eventParams = new ArrayList(); === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java' --- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/EventQueryPlannerTest.java 2014-03-24 22:19:59 +0000 @@ -34,9 +34,11 @@ import java.util.List; import org.hisp.dhis.DhisSpringTest; +import org.hisp.dhis.analytics.Partitions; import org.hisp.dhis.analytics.event.EventQueryParams; import org.hisp.dhis.analytics.event.EventQueryPlanner; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Cal; import org.hisp.dhis.program.Program; import org.junit.Test; @@ -55,21 +57,27 @@ @Autowired private EventQueryPlanner queryPlanner; + @Autowired + private OrganisationUnitService organisationUnitService; + @Override public void setUpTest() { prA = new Program(); - prA.setUid( "programuidA" ); + prA.setUid( "programuida" ); ouA = createOrganisationUnit( 'A' ); ouB = createOrganisationUnit( 'B' ); + organisationUnitService.addOrganisationUnit( ouA ); + organisationUnitService.addOrganisationUnit( ouB ); + ouA.setLevel( 1 ); ouB.setLevel( 2 ); } @Test - public void testPlanQueryA() + public void testPlanAggregateQueryA() { EventQueryParams params = new EventQueryParams(); params.setProgram( prA ); @@ -77,24 +85,25 @@ params.setEndDate( new Cal( 2012, 3, 20 ).time() ); params.setOrganisationUnits( Arrays.asList( ouA ) ); - List queries = queryPlanner.planQuery( params, null ); - - assertEquals( 3, queries.size() ); - - assertEquals( new Cal( 2010, 6, 1 ).time(), queries.get( 0 ).getStartDate() ); - assertEquals( new Cal( 2010, 12, 31 ).time(), queries.get( 0 ).getEndDate() ); - assertEquals( new Cal( 2011, 1, 1 ).time(), queries.get( 1 ).getStartDate() ); - assertEquals( new Cal( 2011, 12, 31 ).time(), queries.get( 1 ).getEndDate() ); - assertEquals( new Cal( 2012, 1, 1 ).time(), queries.get( 2 ).getStartDate() ); - assertEquals( new Cal( 2012, 3, 20 ).time(), queries.get( 2 ).getEndDate() ); - - assertEquals( "analytics_event_2010_programuidA", queries.get( 0 ).getPartitions().getSinglePartition() ); - assertEquals( "analytics_event_2011_programuidA", queries.get( 1 ).getPartitions().getSinglePartition() ); - assertEquals( "analytics_event_2012_programuidA", queries.get( 2 ).getPartitions().getSinglePartition() ); + List queries = queryPlanner.planAggregateQuery( params ); + + assertEquals( 1, queries.size() ); + + EventQueryParams query = queries.get( 0 ); + + assertEquals( new Cal( 2010, 6, 1 ).time(), query.getStartDate() ); + assertEquals( new Cal( 2012, 3, 20 ).time(), query.getEndDate() ); + + Partitions partitions = query.getPartitions(); + + assertEquals( 3, partitions.getPartitions().size() ); + assertEquals( "analytics_event_2010_programuida", partitions.getPartitions().get( 0 ) ); + assertEquals( "analytics_event_2011_programuida", partitions.getPartitions().get( 1 ) ); + assertEquals( "analytics_event_2012_programuida", partitions.getPartitions().get( 2 ) ); } @Test - public void testPlanQueryB() + public void testPlanAggregateQueryB() { EventQueryParams params = new EventQueryParams(); params.setProgram( prA ); @@ -102,18 +111,23 @@ params.setEndDate( new Cal( 2010, 9, 20 ).time() ); params.setOrganisationUnits( Arrays.asList( ouA ) ); - List queries = queryPlanner.planQuery( params, null ); + List queries = queryPlanner.planAggregateQuery( params ); assertEquals( 1, queries.size() ); + + EventQueryParams query = queries.get( 0 ); - assertEquals( new Cal( 2010, 3, 1 ).time(), queries.get( 0 ).getStartDate() ); - assertEquals( new Cal( 2010, 9, 20 ).time(), queries.get( 0 ).getEndDate() ); - - assertEquals( "analytics_event_2010_programuidA", queries.get( 0 ).getPartitions().getSinglePartition() ); - } + assertEquals( new Cal( 2010, 3, 1 ).time(), query.getStartDate() ); + assertEquals( new Cal( 2010, 9, 20 ).time(), query.getEndDate() ); + + Partitions partitions = query.getPartitions(); + + assertEquals( 1, partitions.getPartitions().size() ); + assertEquals( "analytics_event_2010_programuida", partitions.getSinglePartition() ); + } @Test - public void testPlanQueryC() + public void testPlanAggregateQueryC() { EventQueryParams params = new EventQueryParams(); params.setProgram( prA ); @@ -121,8 +135,52 @@ params.setEndDate( new Cal( 2012, 3, 20 ).time() ); params.setOrganisationUnits( Arrays.asList( ouA, ouB ) ); - List queries = queryPlanner.planQuery( params, null ); - - assertEquals( 6, queries.size() ); + List queries = queryPlanner.planAggregateQuery( params ); + + assertEquals( 2, queries.size() ); + assertEquals( ouA, queries.get( 0 ).getOrganisationUnits().get( 0 ) ); + assertEquals( ouB, queries.get( 1 ).getOrganisationUnits().get( 0 ) ); + } + + @Test + public void testPlanEventQueryA() + { + EventQueryParams params = new EventQueryParams(); + params.setProgram( prA ); + params.setStartDate( new Cal( 2010, 6, 1 ).time() ); + params.setEndDate( new Cal( 2012, 3, 20 ).time() ); + params.setOrganisationUnits( Arrays.asList( ouA ) ); + + params = queryPlanner.planEventQuery( params ); + + assertEquals( new Cal( 2010, 6, 1 ).time(), params.getStartDate() ); + assertEquals( new Cal( 2012, 3, 20 ).time(), params.getEndDate() ); + + Partitions partitions = params.getPartitions(); + + assertEquals( 3, partitions.getPartitions().size() ); + assertEquals( "analytics_event_2010_programuida", partitions.getPartitions().get( 0 ) ); + assertEquals( "analytics_event_2011_programuida", partitions.getPartitions().get( 1 ) ); + assertEquals( "analytics_event_2012_programuida", partitions.getPartitions().get( 2 ) ); + } + + @Test + public void testPlanEventQueryB() + { + EventQueryParams params = new EventQueryParams(); + params.setProgram( prA ); + params.setStartDate( new Cal( 2010, 3, 1 ).time() ); + params.setEndDate( new Cal( 2010, 9, 20 ).time() ); + params.setOrganisationUnits( Arrays.asList( ouA ) ); + + params = queryPlanner.planEventQuery( params ); + + assertEquals( new Cal( 2010, 3, 1 ).time(), params.getStartDate() ); + assertEquals( new Cal( 2010, 9, 20 ).time(), params.getEndDate() ); + + Partitions partitions = params.getPartitions(); + + assertEquals( 1, partitions.getPartitions().size() ); + assertEquals( "analytics_event_2010_programuida", partitions.getSinglePartition() ); } }