=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2015-03-04 18:36:19 +0000 @@ -31,6 +31,7 @@ import java.util.Map; import java.util.concurrent.Future; +import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.ListMap; import org.hisp.dhis.common.NameableObject; @@ -46,9 +47,11 @@ * asynchronously. The value class can be Double or String. * * @param params the query to retrieve aggregated data for. + * @param maxLimit the max number of records to retrieve. * @return a map. + * @throws IllegalQueryException if query result set exceeds the max limit. */ - Future> getAggregatedDataValues( DataQueryParams params ); + Future> getAggregatedDataValues( DataQueryParams params, int maxLimit ); /** * Inserts entries for the aggregation periods mapped to each data period === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-02-25 15:59:31 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/DataQueryParams.java 2015-03-04 18:36:19 +0000 @@ -28,8 +28,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import static org.hisp.dhis.analytics.AggregationType.AVERAGE_INT_DISAGGREGATION; import static org.hisp.dhis.analytics.AggregationType.AVERAGE_SUM_INT_DISAGGREGATION; -import static org.hisp.dhis.analytics.AggregationType.AVERAGE_INT_DISAGGREGATION; import static org.hisp.dhis.common.DimensionType.DATASET; import static org.hisp.dhis.common.DimensionType.ORGANISATIONUNIT; import static org.hisp.dhis.common.DimensionType.ORGANISATIONUNIT_GROUPSET; @@ -44,7 +44,6 @@ import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.NameableObjectUtils.asList; import static org.hisp.dhis.common.NameableObjectUtils.getList; -import static org.hisp.dhis.system.util.CollectionUtils.emptyIfNull; import java.util.ArrayList; import java.util.Arrays; @@ -65,6 +64,7 @@ import org.hisp.dhis.common.DimensionalObjectUtils; import org.hisp.dhis.common.DisplayProperty; import org.hisp.dhis.common.ListMap; +import org.hisp.dhis.common.MapMap; import org.hisp.dhis.common.NameableObject; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementCategory; @@ -78,7 +78,6 @@ import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.system.util.CollectionUtils; import org.hisp.dhis.system.util.ListUtils; -import org.hisp.dhis.common.MapMap; import org.hisp.dhis.system.util.MathUtils; /** @@ -544,35 +543,7 @@ return null; } - - /** - * Returns the number of dimension option permutations. Merges the three data - * dimensions into one prior to the calculation. - */ - public int getNumberOfDimensionOptionPermutations() - { - int total = 1; - - DataQueryParams query = this.instance(); - - query.getDimensions().add( new BaseDimensionalObject( DATA_X_DIM_ID ) ); - - query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( INDICATOR_DIM_ID ) ) ); - query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( DATAELEMENT_DIM_ID ) ) ); - query.getDimension( DATA_X_DIM_ID ).getItems().addAll( emptyIfNull( query.getDimensionOptions( DATASET_DIM_ID ) ) ); - - query.removeDimension( INDICATOR_DIM_ID ); - query.removeDimension( DATAELEMENT_DIM_ID ); - query.removeDimension( DATASET_DIM_ID ); - - for ( DimensionalObject dim : query.getDimensions() ) - { - total *= Math.max( dim.getItems().size(), 1 ); - } - - return total; - } - + /** * Returns a list of dimensions which occur more than once, not including * the first duplicate. === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-03-04 17:49:02 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2015-03-04 18:36:19 +0000 @@ -737,6 +737,8 @@ int optimalQueries = MathUtils.getWithin( getProcessNo(), 1, MAX_QUERIES ); + int maxLimit = getMaxLimit(); + Timer t = new Timer().start().disablePrint(); DataQueryGroups queryGroups = queryPlanner.planQuery( params, optimalQueries, tableName ); @@ -751,7 +753,7 @@ for ( DataQueryParams query : queries ) { - futures.add( analyticsManager.getAggregatedDataValues( query ) ); + futures.add( analyticsManager.getAggregatedDataValues( query, maxLimit ) ); } for ( Future> future : futures ) @@ -1375,4 +1377,12 @@ { return value != null && Double.class.equals( value.getClass() ) ? MathUtils.getRounded( (Double) value ) : value; } + + /** + * Returns the max records limit. 0 indicates no limit. + */ + private int getMaxLimit() + { + return (Integer) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_ANALYTICS_MAX_LIMIT, SystemSettingManager.DEFAULT_ANALYTICS_MAX_LIMIT ); + } } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2015-02-05 18:58:22 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2015-03-04 18:36:19 +0000 @@ -140,11 +140,6 @@ violation = "Category option combos cannot be specified as filter"; } - if ( !params.isIgnoreLimit() && getMaxLimit() > 0 && params.getNumberOfDimensionOptionPermutations() > getMaxLimit() ) - { - violation = "Table exceeds max number of cells: " + getMaxLimit() + " (" + params.getNumberOfDimensionOptionPermutations() + ")"; - } - if ( !params.getDuplicateDimensions().isEmpty() ) { violation = "Dimensions cannot be specified more than once: " + params.getDuplicateDimensions(); @@ -795,12 +790,4 @@ return map; } - - /** - * Returns the max records limit. 0 indicates no limit. - */ - private int getMaxLimit() - { - return (Integer) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_ANALYTICS_MAX_LIMIT, SystemSettingManager.DEFAULT_ANALYTICS_MAX_LIMIT ); - } } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2015-03-04 18:36:19 +0000 @@ -66,6 +66,7 @@ import org.hisp.dhis.analytics.MeasureFilter; import org.hisp.dhis.common.DimensionalObject; import org.hisp.dhis.common.DimensionalObjectUtils; +import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.ListMap; import org.hisp.dhis.common.NameableObject; import org.hisp.dhis.jdbc.StatementBuilder; @@ -112,7 +113,7 @@ @Override @Async - public Future> getAggregatedDataValues( DataQueryParams params ) + public Future> getAggregatedDataValues( DataQueryParams params, int maxLimit ) { try { @@ -139,7 +140,7 @@ try { - map = getKeyValueMap( params, sql ); + map = getKeyValueMap( params, sql, maxLimit ); } catch ( BadSqlGrammarException ex ) { @@ -389,8 +390,7 @@ * Retrieves data from the database based on the given query and SQL and puts * into a value key and value mapping. */ - private Map getKeyValueMap( DataQueryParams params, String sql ) - throws BadSqlGrammarException + private Map getKeyValueMap( DataQueryParams params, String sql, int maxLimit ) { Map map = new HashMap<>(); @@ -398,8 +398,15 @@ log.debug( "Analytics SQL: " + sql ); + int counter = 0; + while ( rowSet.next() ) { + if ( maxLimit > 0 && ++counter > maxLimit ) + { + throw new IllegalQueryException( "Query result set exceeds max limit: " + maxLimit ); + } + StringBuilder key = new StringBuilder(); for ( DimensionalObject dim : params.getQueryDimensions() ) === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java' --- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/QueryPlannerTest.java 2015-03-04 18:36:19 +0000 @@ -337,29 +337,6 @@ assertEquals( PERIOD_DIM_ID, permutation.get( 1 ).getDimension() ); } } - - /** - * First, combines data elements and data sets into one data dimension and - * returns (2 + 3) * 3 * 2 = 30. Second, ignores any data dimension and - * returns 3 * 2 = 6. - */ - @Test - public void testGetNumberOfDimensionOptionPermutations() - { - DataQueryParams params = new DataQueryParams(); - params.setDataElements( getList( deA, deB ) ); - params.setDataSets( getList( dsA, dsB, dsC ) ); - params.setOrganisationUnits( getList( ouA, ouB, ouC ) ); - params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ) ) ); - - assertEquals( 30, params.getNumberOfDimensionOptionPermutations() ); - - params = new DataQueryParams(); - params.setOrganisationUnits( getList( ouA, ouB, ouC ) ); - params.setPeriods( getList( createPeriod( "2000Q1" ), createPeriod( "2000Q2" ) ) ); - - assertEquals( 6, params.getNumberOfDimensionOptionPermutations() ); - } @Test public void testGetDataPeriodAggregationPeriodMap()