=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2016-01-05 12:41:21 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2016-01-13 12:54:38 +0000 @@ -72,7 +72,6 @@ String DAYS_SYMBOL = "[days]"; String VARIABLE_EXPRESSION = "(#|D|A|I)\\{(([a-zA-Z]\\w{10})\\.?(\\w*))\\}"; - String AGGREGATE_EXPRESSION = "#\\[([^\\]]*)\\]"; String OPERAND_EXPRESSION = "#\\{([a-zA-Z]\\w{10})\\.?(\\w*)\\}"; String PROGRAM_DATA_ELEMENT_EXPRESSION = "D\\{([a-zA-Z]\\w{10})\\.?([a-zA-Z]\\w{10})\\}"; String OPERAND_UID_EXPRESSION = "([a-zA-Z]\\w{10})\\.?(\\w*)"; @@ -83,7 +82,6 @@ String DAYS_EXPRESSION = "\\[days\\]"; Pattern VARIABLE_PATTERN = Pattern.compile( VARIABLE_EXPRESSION ); - Pattern AGGREGATE_PATTERN = Pattern.compile(AGGREGATE_EXPRESSION); Pattern OPERAND_PATTERN = Pattern.compile( OPERAND_EXPRESSION ); Pattern OPERAND_UID_PATTERN = Pattern.compile( OPERAND_UID_EXPRESSION ); Pattern PROGRAM_DATA_ELEMENT_PATTERN = Pattern.compile( PROGRAM_DATA_ELEMENT_EXPRESSION ); === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRule.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRule.java 2016-01-12 00:42:41 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRule.java 2016-01-13 12:54:38 +0000 @@ -82,7 +82,8 @@ private RuleType ruleType = RuleType.VALIDATION; /** - * The comparison operator to compare left and right expressions in the rule. + * The comparison operator to compare left and right expressions in the + * rule. */ private Operator operator; @@ -107,20 +108,22 @@ private Set groups = new HashSet<>(); /** - * The organisation unit level at which this rule is evaluated (Monitoring-type rules only). + * The organisation unit level at which this rule is evaluated + * (Monitoring-type rules only). */ private Integer organisationUnitLevel; /** * The number of sequential right-side periods from which to collect samples * to average (Monitoring-type rules only). Sequential periods are those - * immediately preceding (or immediately following in previous years) the selected period. + * immediately preceding (or immediately following in previous years) the + * selected period. */ private Integer sequentialSampleCount; /** - * The number of annual right-side periods from which to collect samples - * to average (Monitoring-type rules only). Annual periods are from previous + * The number of annual right-side periods from which to collect samples to + * average (Monitoring-type rules only). Annual periods are from previous * years. Samples collected from previous years can also include sequential * periods adjacent to the equivalent period in previous years. */ @@ -135,8 +138,8 @@ } - public ValidationRule( String name, String description, - Operator operator, Expression leftSide, Expression rightSide ) + public ValidationRule( String name, String description, Operator operator, Expression leftSide, + Expression rightSide ) { this.name = name; this.description = description; @@ -150,9 +153,9 @@ // ------------------------------------------------------------------------- /** - * Clears the left-side and right-side expressions. This can be useful, for example, - * before changing the validation rule period type, because the data elements - * allowed in the expressions depend on the period type. + * Clears the left-side and right-side expressions. This can be useful, for + * example, before changing the validation rule period type, because the + * data elements allowed in the expressions depend on the period type. */ public void clearExpressions() { @@ -183,8 +186,8 @@ } /** - * Gets the validation rule description, but returns the validation rule name - * if there is no description. + * Gets the validation rule description, but returns the validation rule + * name if there is no description. * * @return the description (or name). */ @@ -194,9 +197,9 @@ } /** - * Gets the data elements to evaluate for the current period. For validation-type - * rules this means all data elements. For monitoring-type rules this means just - * the left side elements. + * Gets the data elements to evaluate for the current period. For + * validation-type rules this means all data elements. For monitoring-type + * rules this means just the left side elements. * * @return the data elements to evaluate for the current period. */ @@ -214,9 +217,9 @@ } /** - * Gets the data elements to compare against for past periods. For validation-type - * rules this returns null. For monitoring-type rules this is just the - * right side elements. + * Gets the data elements to compare against for past periods. For + * validation-type rules this returns null. For monitoring-type rules this + * is just the right side elements. * * @return the data elements to evaluate for past periods. */ @@ -254,7 +257,8 @@ } else if ( leftSide != null && rightSide != null ) { - return leftSide.getDescription() + " " + operator.getMathematicalOperator() + " " + rightSide.getDescription(); + return leftSide.getDescription() + " " + operator.getMathematicalOperator() + " " + + rightSide.getDescription(); } else { @@ -376,7 +380,6 @@ this.annualSampleCount = annualSampleCount; } - @JsonProperty @JsonView( { DetailedView.class, ExportView.class } ) @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 2016-01-04 02:27:49 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 2016-01-13 12:54:38 +0000 @@ -54,17 +54,18 @@ public class Validator { /** - * Evaluates validation rules for a collection of organisation units. - * This method breaks the job down by organisation unit. It assigns the + * Evaluates validation rules for a collection of organisation units. This + * method breaks the job down by organisation unit. It assigns the * evaluation for each organisation unit to a task that can be evaluated * independently in a multi-threaded environment. * - * @param sources the organisation units in which to run the validation rules + * @param sources the organisation units in which to run the validation + * rules * @param periods the periods of data to check * @param attributeCombo the attribute combo to check (if restricted) * @param rules the ValidationRules to evaluate - * @param lastScheduledRun date/time of the most recent successful - * scheduled monitoring run (needed only for scheduled runs) + * @param lastScheduledRun date/time of the most recent successful scheduled + * monitoring run (needed only for scheduled runs) * @param constantService Constant Service reference * @param expressionService Expression Service reference * @param periodService Period Service reference @@ -74,15 +75,16 @@ * @param currentUserService current user service * @return a collection of any validations that were found */ - public static Collection validate( Collection sources, Collection periods, - Collection rules, DataElementCategoryOptionCombo attributeCombo, Date lastScheduledRun, - ConstantService constantService, ExpressionService expressionService, PeriodService periodService, - DataValueService dataValueService, DataElementCategoryService dataElementCategoryService, - UserService userService, CurrentUserService currentUserService ) + public static Collection validate( Collection sources, + Collection periods, Collection rules, DataElementCategoryOptionCombo attributeCombo, + Date lastScheduledRun, ConstantService constantService, ExpressionService expressionService, + PeriodService periodService, DataValueService dataValueService, + DataElementCategoryService dataElementCategoryService, UserService userService, + CurrentUserService currentUserService ) { - ValidationRunContext context = ValidationRunContext.getNewContext( sources, periods, - attributeCombo, rules, constantService.getConstantMap(), ValidationRunType.SCHEDULED, lastScheduledRun, - expressionService, periodService, dataValueService, dataElementCategoryService, userService, currentUserService ); + ValidationRunContext context = ValidationRunContext.getNewContext( sources, periods, attributeCombo, rules, + constantService.getConstantMap(), ValidationRunType.SCHEDULED, lastScheduledRun, expressionService, + periodService, dataValueService, dataElementCategoryService, userService, currentUserService ); int threadPoolSize = getThreadPoolSize( context ); ExecutorService executor = Executors.newFixedThreadPool( threadPoolSize ); @@ -97,7 +99,7 @@ } executor.shutdown(); - + try { executor.awaitTermination( 6, TimeUnit.HOURS ); @@ -111,7 +113,7 @@ return context.getValidationResults(); } - + /** * Determines how many threads we should use for testing validation rules. * @@ -121,18 +123,18 @@ private static int getThreadPoolSize( ValidationRunContext context ) { int threadPoolSize = SystemUtils.getCpuCores(); - + if ( threadPoolSize > 2 ) { threadPoolSize--; } - + if ( threadPoolSize > context.getCountOfSourcesToValidate() ) { threadPoolSize = context.getCountOfSourcesToValidate(); } - return threadPoolSize; + return threadPoolSize; } /** @@ -141,11 +143,13 @@ * @param results * @param dataElementCategoryService */ - private static void reloadAttributeOptionCombos( Collection results, DataElementCategoryService dataElementCategoryService ) + private static void reloadAttributeOptionCombos( Collection results, + DataElementCategoryService dataElementCategoryService ) { for ( ValidationResult result : results ) { - result.setAttributeOptionCombo( dataElementCategoryService.getDataElementCategoryOptionCombo( result.getAttributeOptionCombo().getId() ) ); + result.setAttributeOptionCombo( dataElementCategoryService + .getDataElementCategoryOptionCombo( result.getAttributeOptionCombo().getId() ) ); } } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2016-01-06 17:14:10 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2016-01-13 12:54:38 +0000 @@ -55,6 +55,7 @@ import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.system.util.MathUtils; + /** * Runs a validation task on a thread within a multi-threaded validation run. *

@@ -63,609 +64,597 @@ * @author Jim Grace */ public class ValidatorThread -implements Runnable + implements Runnable { - private static final Log log = LogFactory.getLog( ValidatorThread.class ); - - private OrganisationUnitExtended sourceX; - - private ValidationRunContext context; - - public ValidatorThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) - { - this.sourceX = sourceX; - this.context = context; - } - - /** - * Evaluates validation rules for a single organisation unit. This is the - * central method in validation rule evaluation. - */ - @Override - public void run() - { - try - { - runInternal(); - } - catch ( RuntimeException ex ) - { - log.error( DebugUtils.getStackTrace( ex ) ); - - throw ex; - } - } - - private void runInternal() - { - if ( context.getValidationResults().size() < (ValidationRunType.INTERACTIVE == context.getRunType() ? - ValidationRuleService.MAX_INTERACTIVE_ALERTS : ValidationRuleService.MAX_SCHEDULED_ALERTS) ) - { - for ( PeriodTypeExtended periodTypeX : context.getPeriodTypeExtendedMap().values() ) - { - Collection sourceDataElements = periodTypeX.getSourceDataElements().get( sourceX.getSource() ); - Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements ); - context.getExpressionService().explodeValidationRuleExpressions( rules ); - - if ( !rules.isEmpty() ) - { - Set recursiveCurrentDataElements = getRecursiveCurrentDataElements( rules ); - - for ( Period period : periodTypeX.getPeriods() ) - { - MapMap lastUpdatedMap = new MapMap<>(); - SetMap incompleteValuesMap = new SetMap<>(); - MapMap currentValueMap = getValueMap - ( periodTypeX, periodTypeX.getDataElements(), sourceDataElements, recursiveCurrentDataElements, - periodTypeX.getAllowedPeriodTypes(), period, sourceX.getSource(), lastUpdatedMap, incompleteValuesMap ); - - log.trace( "Source " + sourceX.getSource().getName() - + " [" + period.getStartDate() + " - " + period.getEndDate() + "]" - + " currentValueMap[" + currentValueMap.size() + "]" ); - - for ( ValidationRule rule : rules ) - { - if ( evaluateValidationCheck( currentValueMap, lastUpdatedMap, rule ) ) - { - int n_years = rule.getAnnualSampleCount() == null ? 0 : rule.getAnnualSampleCount(); - int window = rule.getSequentialSampleCount() == null ? 0 : rule - .getSequentialSampleCount(); - Collection periodTypes = context.getRuleXMap().get(rule).getAllowedPastPeriodTypes(); - - log.debug("Rule "+rule.getName()+" @"+period.getDisplayShortName()+" & "+sourceX.getSource()+ - " window="+window+", years="+n_years); - Map leftSideValues = getRuleExpressionValueMap - ( rule.getLeftSide(),currentValueMap, incompleteValuesMap, - sourceX.getSource(), period, window, n_years, periodTypeX, - periodTypes, lastUpdatedMap, sourceDataElements ); - - if ( !leftSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - Map rightSideValues = - getRuleExpressionValueMap - ( rule.getRightSide(),currentValueMap, incompleteValuesMap, - sourceX.getSource(), period, window, n_years, periodTypeX, - periodTypes, lastUpdatedMap, sourceDataElements ); - - if ( !rightSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - Set attributeOptionCombos = leftSideValues.keySet(); - - if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - attributeOptionCombos = new HashSet<>( attributeOptionCombos ); - attributeOptionCombos.addAll( rightSideValues.keySet() ); - } - - for ( int optionCombo : attributeOptionCombos ) - { - Double leftSide = leftSideValues.get( optionCombo ); - Double rightSide = rightSideValues.get( optionCombo ); - boolean violation = false; - - if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - violation = (leftSide != null && rightSide == null) - || (leftSide == null && rightSide != null); - } - else if ( leftSide != null && rightSide != null ) - { - violation = !expressionIsTrue( leftSide, rule.getOperator(), rightSide ); - } - - if ( violation ) - { - context.getValidationResults().add - (new ValidationResult - (period, sourceX.getSource(), - context.getDataElementCategoryService().getDataElementCategoryOptionCombo( optionCombo ), rule, - roundSignificant( zeroIfNull( leftSide ) ), - roundSignificant( zeroIfNull( rightSide ) ) ) ); - } - - log.debug( "Evaluated " + rule.getName() - + ", combo id " + optionCombo + ": " - + (violation ? "violation" : "OK") + " " + (leftSide == null ? "(null)" : leftSide.toString()) - + " " + rule.getOperator() + " " + (rightSide == null ? "(null)" : rightSide.toString()) - + " (" + context.getValidationResults().size() + " results)" ); - - } - } - } - } - } - } - } - } - } - } - - /** - * Gets the rules that should be evaluated for a given organisation unit and - * period type. - * - * @param sourceX the organisation unit extended information - * @param periodTypeX the period type extended information - * @param sourceDataElements all data elements collected for this - * organisation unit - * @return set of rules for this org unit and period type - */ - private Set getRulesBySourceAndPeriodType( OrganisationUnitExtended sourceX, - PeriodTypeExtended periodTypeX, Collection sourceDataElements ) - { - Set periodTypeRules = new HashSet<>(); - - for ( ValidationRule rule : periodTypeX.getRules() ) - { - if ( rule.getRuleType() == RuleType.VALIDATION ) - { - // For validation-type rules, include only rules where the - // organisation collects all the data elements in the rule. - // But if this is some funny kind of rule with no elements - // (like for testing), include it also. - Collection elements = rule.getCurrentDataElements(); - - if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) ) - { - periodTypeRules.add( rule ); - } - } - else - { - // For surveillance-type rules, include only rules for this - // organisation's unit level. - // The organisation may not be configured for the data elements - // because they could be aggregated from a lower level. - if ( rule.getOrganisationUnitLevel() == sourceX.getLevel() ) - { - periodTypeRules.add( rule ); - } - } - } - - return periodTypeRules; - } - - /** - * Checks to see if the evaluation should go further for this - * evaluationRule, after the "current" data to evaluate has been fetched. - * For INTERACTIVE runs, we always go further (always return true.) For - * SCHEDULED runs, we go further only if something has changed since the - * last successful scheduled run -- either the rule definition or one of - * the "current" data element / option values on the left or right sides. - *

- * For scheduled runs, remove all values for any attribute option combos - * where nothing has changed since the last run. - * - * @param lastUpdatedMapMap when each data value was last updated - * @param rule the rule that may be evaluated - * @return true if the rule should be evaluated with this data, false if not - */ - private boolean evaluateValidationCheck( MapMap currentValueMapMap, - MapMap lastUpdatedMapMap, ValidationRule rule ) - { - boolean evaluate = true; // Assume true for now. - - if ( ValidationRunType.SCHEDULED == context.getRunType() ) - { - if ( context.getLastScheduledRun() != null ) // True if no previous scheduled run - { - if ( rule.getLastUpdated().before( context.getLastScheduledRun() ) ) - { - // Get the "current" DataElementOperands from this rule: - // Left+Right sides for VALIDATION, Left side only for - // SURVEILLANCE. - Collection deos = context.getExpressionService().getOperandsInExpression( - rule.getLeftSide().getExpression() ); - - if ( rule.getRuleType() == RuleType.VALIDATION ) - { - // Make a copy so we can add to it. - deos = new HashSet<>( deos ); - deos.addAll( context.getExpressionService().getOperandsInExpression( rule.getRightSide().getExpression() ) ); - } - - // Return true if any data is more recent than the last - // scheduled run, otherwise return false. - evaluate = false; - - for ( Map.Entry> entry : lastUpdatedMapMap.entrySet() ) - { - boolean saveThisCombo = false; - - for ( DataElementOperand deo : deos ) - { - Date lastUpdated = entry.getValue().get( deo ); - - if ( lastUpdated != null && lastUpdated.after( context.getLastScheduledRun() ) ) - { - saveThisCombo = true; // True if new/updated data. - evaluate = true; - break; - } - } - - if ( !saveThisCombo ) - { - currentValueMapMap.remove( entry.getKey() ); - } - } - } - } - } - return evaluate; - } - - /** - * Gets the data elements for which values should be fetched recursively if - * they are not collected for an organisation unit. - * - * @param rules ValidationRules to be evaluated - * @return the data elements to fetch recursively - */ - private Set getRecursiveCurrentDataElements( Set rules ) - { - Set recursiveCurrentDataElements = new HashSet<>(); - - for ( ValidationRule rule : rules ) - { - if ( rule.getRuleType() == RuleType.SURVEILLANCE && rule.getCurrentDataElements() != null ) - { - recursiveCurrentDataElements.addAll( rule.getCurrentDataElements() ); - } - } - - return recursiveCurrentDataElements; - } - - - /** - * Evaluates an expression, returning a map of values by attribute option - * combo. - * - * @param expression expression to evaluate. - * @param valueMap Map of value maps, by attribute option combo. - * @param incompleteValuesMap map of values that were incomplete. - * @return map of values. - */ - private Map getExpressionValueMap( Expression expression, - MapMap valueMap, - SetMap incompleteValuesMap ) - { - Map expressionValueMap = new HashMap<>(); - - for ( Map.Entry> entry : valueMap.entrySet() ) - { - Double value = context.getExpressionService().getExpressionValue( expression, - entry.getValue(), context.getConstantMap(), null, null, - incompleteValuesMap.getSet( entry.getKey() ), null ); - - if ( MathUtils.isValidDouble( value ) ) - { - expressionValueMap.put( entry.getKey(), value ); - } - } - - return expressionValueMap; - } - - /** - * Gets data values for a given organisation unit and period, recursing if - * necessary to sum the values from child organisation units. - * - * @param periodTypeX period type which we are evaluating - * @param ruleDataElements data elements configured for the rule - * @param sourceDataElements data elements configured for the organisation - * unit - * @param recursiveDataElements data elements for which we will recurse if - * necessary - * @param allowedPeriodTypes all the periods in which we might find the data - * values - * @param period period in which we are looking for values - * @param source organisation unit for which we are looking for values - * @param lastUpdatedMap map showing when each data values was last updated - * @param incompleteValuesMap ongoing set showing which values were found - * but not from all children, mapped by attribute option combo. - * @return map of attribute option combo to map of values found. - */ - private MapMap getValueMap( PeriodTypeExtended periodTypeX, - Collection ruleDataElements, Collection sourceDataElements, - Set recursiveDataElements, Collection allowedPeriodTypes, Period period, - OrganisationUnit source, MapMap lastUpdatedMap, - SetMap incompleteValuesMap ) - { - Set dataElementsToGet = new HashSet<>( ruleDataElements ); - dataElementsToGet.retainAll( sourceDataElements ); - - log.trace( "getDataValueMapRecursive: source:" + source.getName() - + " ruleDataElements[" + ruleDataElements.size() - + "] sourceDataElements[" + sourceDataElements.size() - + "] elementsToGet[" + dataElementsToGet.size() - + "] recursiveDataElements[" + recursiveDataElements.size() - + "] allowedPeriodTypes[" + allowedPeriodTypes.size() + "]" ); - - MapMap dataValueMap = null; - - if ( dataElementsToGet.isEmpty() ) - { - // We still might get something recursively - dataValueMap = new MapMap<>(); - } - else - { - dataValueMap = context.getDataValueService().getDataValueMapByAttributeCombo( dataElementsToGet, - period.getStartDate(), source, allowedPeriodTypes, context.getAttributeCombo(), - context.getCogDimensionConstraints(), context.getCoDimensionConstraints(), lastUpdatedMap ); - } - - // See if there are any data elements we need to get recursively: - Set recursiveDataElementsNeeded = new HashSet<>( recursiveDataElements ); - recursiveDataElementsNeeded.removeAll( dataElementsToGet ); - - if ( !recursiveDataElementsNeeded.isEmpty() ) - { - int childCount = 0; - MapMap childValueCounts = new MapMap<>(); - - for ( OrganisationUnit child : source.getChildren() ) - { - Collection childDataElements = periodTypeX.getSourceDataElements().get( child ); - MapMap childMap = getValueMap( periodTypeX, - recursiveDataElementsNeeded, childDataElements, recursiveDataElementsNeeded, allowedPeriodTypes, - period, child, lastUpdatedMap, incompleteValuesMap ); - - for ( Map.Entry> entry : childMap.entrySet() ) - { - int combo = entry.getKey(); - - for ( Map.Entry e : entry.getValue().entrySet() ) - { - DataElementOperand deo = e.getKey(); - Double childValue = e.getValue(); - - Double baseValue = dataValueMap.getValue( combo, deo ); - dataValueMap.putEntry( combo, deo, baseValue == null ? childValue : baseValue + childValue ); - - Integer childValueCount = childValueCounts.getValue( combo, deo ); - childValueCounts.putEntry( combo, deo, childValueCount == null ? 1 : childValueCount + 1 ); - } - } - - childCount++; - } - - for ( Map.Entry> entry : childValueCounts.entrySet() ) - { - int combo = entry.getKey(); - - for ( Map.Entry e : entry.getValue().entrySet() ) - { - DataElementOperand deo = e.getKey(); - Integer childValueCount = e.getValue(); - - if ( childValueCount != childCount ) - { - // Remember that we found this DataElementOperand value - // in some but not all children - incompleteValuesMap.putValue( combo, deo ); - } - } - } - } - - return dataValueMap; - } - - /* Generalized surveillance rules */ - - /** - * Returns the right-side evaluated value of the validation rule. - * - * @param source organisation unit being evaluated - * @param periodTypeX period type being evaluated - * @param period period being evaluated - * @param rule ValidationRule being evaluated - * @param currentValueMap current values already fetched - * @param periodTypes applicable period types - * @param sourceDataElements the data elements collected by the organisation - * unit - * @return the right-side values, map by attribute category combo - */ - private ListMap getAggregateValueMap - (Expression expression,OrganisationUnit source, - Period period,int window, int n_years, - PeriodTypeExtended px, - Collection periodTypes, - MapMap lastUpdatedMap, - Collection sourceDataElements) - { - ListMap results = new ListMap(); - CalendarPeriodType periodType = (CalendarPeriodType) period.getPeriodType(); - Calendar yearly = PeriodType.createCalendarInstance( period.getStartDate() ); - - for ( int years = 0; years <= n_years; years++ ) - { - // Defensive copy because createPeriod mutates Calendar. - Calendar each_year = PeriodType.createCalendarInstance( yearly.getTime() ); - // To track the period at the same time in preceding years. - Period base_period = periodType.createPeriod( each_year ); - - if (years>0) { - // For past years, fetch a window around the period at the - // same time of year as this period. - gatherPeriodValues(results,expression,source, - base_period,0,px, - periodTypes,lastUpdatedMap, - sourceDataElements); - if (window != 0) - gatherPeriodValues(results, expression, source, periodType.getNextPeriod(base_period), window-1, px, - periodTypes, lastUpdatedMap, sourceDataElements); - } - if (window != 0) - gatherPeriodValues(results, expression, source, periodType.getPreviousPeriod(base_period), 1-window, px, - periodTypes, lastUpdatedMap, sourceDataElements); - - // Move to the previous year. - yearly.set( Calendar.YEAR, yearly.get( Calendar.YEAR ) - 1 ); - } - - return results; - } - - /** - * Gathers the values of an expression for a given organisation unit - and period, accumulating a range of values around the given period. - *

- * Note that for a surveillance-type rule, evaluating the right side - * expression can result in sampling multiple periods and/or child - * organisation units. - * - * @param results the ListMap into which results will be stored - * @param expression the expression to be evaluated - * @param source the organisation unit - * @param period the main period for the validation rule evaluation - * @param window how many periods (before and after) to collect - * @param px the period type extended information - * @param periodTypes the period types in which the data may exist - * @param sourceElements the data elements configured for this - * organisation unit - */ - private void gatherPeriodValues( ListMap results, - Expression expression, - OrganisationUnit source, - Period period,int window, - PeriodTypeExtended px, - Collection periodTypes, - MapMap lastUpdatedMap, - Collection sourceElements) - { - CalendarPeriodType periodType = (CalendarPeriodType) period.getPeriodType(); - Period periodInstance = context.getPeriodService().getPeriod(period.getStartDate(), period.getEndDate(), - periodType); - if (periodInstance == null) - return; - Set dataElements = getExpressionDataElements(expression); - SetMap incompleteValuesMap = new SetMap<>(); - MapMap dataValueMapByAttributeCombo = getValueMap(px, dataElements, - sourceElements, dataElements, periodTypes, periodInstance, source, lastUpdatedMap, incompleteValuesMap); - Map eValues = getExpressionValueMap(expression, dataValueMapByAttributeCombo, - incompleteValuesMap); - int direction = ((window < 0) ? (-1) : (window > 0) ? (1) : (0)); - int steps = ((direction > 0) ? (window) : (direction < 0) ? (-window) : (0)); - results.putValueMap(eValues); - log.debug("Gathering '" + expression.getExpression() + "' " + "at " + period + " (" + window + ") " - + "from " + source.getName() + " starting with:\n\t" + eValues); - if (direction == 0) - return; - Period scan = new Period(periodInstance); - for (int count = 0; count < steps; count++) { - if (direction < 0) - scan = periodType.getPreviousPeriod(scan); - else - scan = periodType.getNextPeriod(scan); - gatherPeriodValues(results, expression, source, scan, 0, px, - periodTypes, lastUpdatedMap, sourceElements); - } - } - - /** - * Returns the data elements referenced in an expression, as a set. - * - * This will return an empty set if e.getDataElementsInExpression returns null - * - * @param expression expression to evaluate. - * @return a Set of DataElement(s) - */ - private Set getExpressionDataElements(Expression e) - { - Set elts=e.getDataElementsInExpression(); - if (elts==null) - return new HashSet(); - else return elts; - } - - /** - * Evaluates an expression, returning a map of values by attribute option - * combo. - * - * @param expression expression to evaluate. - * @param valueMap Map of value maps, by attribute option combo. - * @param incompleteValuesMap map of values that were incomplete. - * @return map of values. - */ - private Map getRuleExpressionValueMap( Expression expression, - MapMap valueMap, - SetMap incompleteValuesMap, - OrganisationUnit source, - Period period,int window,int n_years, - PeriodTypeExtended px, - Collection periodTypes, - MapMap lastUpdatedMap, - Collection sourceElements) - { - Map expressionValueMap = new HashMap<>(); - Map> aggregateValuesMap = new HashMap<>(); - Set aggregates=context.getExpressionService().getAggregatesInExpression(expression.getExpression()); - - if (aggregates.size()==0) - return getExpressionValueMap(expression,valueMap,incompleteValuesMap); - - for (String subExpression: aggregates) - { - Expression subexp=new Expression(subExpression,"aggregated", - new HashSet(sourceElements)); - ListMap aggregateValues=getAggregateValueMap - (subexp,source,period,window,n_years, - px,periodTypes,lastUpdatedMap,sourceElements); - for (Integer attributeOptionCombo: aggregateValues.keySet()) - { - ListMap aggmap; - if (aggregateValuesMap.containsKey(attributeOptionCombo)) - aggmap=aggregateValuesMap.get(attributeOptionCombo); - else { - aggmap=new ListMap<>(); - aggregateValuesMap.put(attributeOptionCombo, aggmap);} - - aggmap.put(subExpression, aggregateValues.get(attributeOptionCombo)); - } - } - - for ( Map.Entry> entry : valueMap.entrySet() ) - { - Double value = context.getExpressionService().getExpressionValue( expression, - entry.getValue(), context.getConstantMap(), null, null, - incompleteValuesMap.getSet( entry.getKey() ), - aggregateValuesMap.get(entry.getKey())); - - if ( MathUtils.isValidDouble( value ) ) - { - expressionValueMap.put( entry.getKey(), value ); - } - } - - return expressionValueMap; - } - - + private static final Log log = LogFactory.getLog( ValidatorThread.class ); + + private OrganisationUnitExtended sourceX; + + private ValidationRunContext context; + + public ValidatorThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) + { + this.sourceX = sourceX; + this.context = context; + } + + /** + * Evaluates validation rules for a single organisation unit. This is the + * central method in validation rule evaluation. + */ + @Override + public void run() + { + try + { + runInternal(); + } + catch ( RuntimeException ex ) + { + log.error( DebugUtils.getStackTrace( ex ) ); + + throw ex; + } + } + + private void runInternal() + { + if ( context.getValidationResults().size() < (ValidationRunType.INTERACTIVE == context.getRunType() + ? ValidationRuleService.MAX_INTERACTIVE_ALERTS : ValidationRuleService.MAX_SCHEDULED_ALERTS) ) + { + for ( PeriodTypeExtended periodTypeX : context.getPeriodTypeExtendedMap().values() ) + { + Collection sourceDataElements = periodTypeX.getSourceDataElements() + .get( sourceX.getSource() ); + Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements ); + context.getExpressionService().explodeValidationRuleExpressions( rules ); + + if ( !rules.isEmpty() ) + { + Set recursiveCurrentDataElements = getRecursiveCurrentDataElements( rules ); + + for ( Period period : periodTypeX.getPeriods() ) + { + MapMap lastUpdatedMap = new MapMap<>(); + SetMap incompleteValuesMap = new SetMap<>(); + MapMap currentValueMap = getValueMap( periodTypeX, + periodTypeX.getDataElements(), sourceDataElements, recursiveCurrentDataElements, + periodTypeX.getAllowedPeriodTypes(), period, sourceX.getSource(), lastUpdatedMap, + incompleteValuesMap ); + + log.trace( "Source " + sourceX.getSource().getName() + " [" + period.getStartDate() + " - " + + period.getEndDate() + "]" + " currentValueMap[" + currentValueMap.size() + "]" ); + + for ( ValidationRule rule : rules ) + { + if ( evaluateValidationCheck( currentValueMap, lastUpdatedMap, rule ) ) + { + int n_years = rule.getAnnualSampleCount() == null ? 0 : rule.getAnnualSampleCount(); + int window = rule.getSequentialSampleCount() == null ? 0 + : rule.getSequentialSampleCount(); + Collection periodTypes = context.getRuleXMap().get( rule ) + .getAllowedPastPeriodTypes(); + + log.debug( "Rule " + rule.getName() + " @" + period.getDisplayShortName() + " & " + + sourceX.getSource() + " window=" + window + ", years=" + n_years ); + Map leftSideValues = getRuleExpressionValueMap( rule.getLeftSide(), + currentValueMap, incompleteValuesMap, sourceX.getSource(), period, window, n_years, + periodTypeX, periodTypes, lastUpdatedMap, sourceDataElements ); + + if ( !leftSideValues.isEmpty() + || Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + Map rightSideValues = getRuleExpressionValueMap( + rule.getRightSide(), currentValueMap, incompleteValuesMap, sourceX.getSource(), + period, window, n_years, periodTypeX, periodTypes, lastUpdatedMap, + sourceDataElements ); + + if ( !rightSideValues.isEmpty() + || Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + Set attributeOptionCombos = leftSideValues.keySet(); + + if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + attributeOptionCombos = new HashSet<>( attributeOptionCombos ); + attributeOptionCombos.addAll( rightSideValues.keySet() ); + } + + for ( int optionCombo : attributeOptionCombos ) + { + Double leftSide = leftSideValues.get( optionCombo ); + Double rightSide = rightSideValues.get( optionCombo ); + boolean violation = false; + + if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + violation = (leftSide != null && rightSide == null) + || (leftSide == null && rightSide != null); + } + else if ( leftSide != null && rightSide != null ) + { + violation = !expressionIsTrue( leftSide, rule.getOperator(), + rightSide ); + } + + if ( violation ) + { + context.getValidationResults() + .add( new ValidationResult( period, sourceX.getSource(), + context.getDataElementCategoryService() + .getDataElementCategoryOptionCombo( optionCombo ), + rule, roundSignificant( zeroIfNull( leftSide ) ), + roundSignificant( zeroIfNull( rightSide ) ) ) ); + } + + log.debug( "Evaluated " + rule.getName() + ", combo id " + optionCombo + + ": " + (violation ? "violation" : "OK") + " " + + (leftSide == null ? "(null)" : leftSide.toString()) + " " + + rule.getOperator() + " " + + (rightSide == null ? "(null)" : rightSide.toString()) + " (" + + context.getValidationResults().size() + " results)" ); + + } + } + } + } + } + } + } + } + } + } + + /** + * Gets the rules that should be evaluated for a given organisation unit and + * period type. + * + * @param sourceX the organisation unit extended information + * @param periodTypeX the period type extended information + * @param sourceDataElements all data elements collected for this + * organisation unit + * @return set of rules for this org unit and period type + */ + private Set getRulesBySourceAndPeriodType( OrganisationUnitExtended sourceX, + PeriodTypeExtended periodTypeX, Collection sourceDataElements ) + { + Set periodTypeRules = new HashSet<>(); + + for ( ValidationRule rule : periodTypeX.getRules() ) + { + if ( rule.getRuleType() == RuleType.VALIDATION ) + { + // For validation-type rules, include only rules where the + // organisation collects all the data elements in the rule. + // But if this is some funny kind of rule with no elements + // (like for testing), include it also. + Collection elements = rule.getCurrentDataElements(); + + if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) ) + { + periodTypeRules.add( rule ); + } + } + else + { + // For surveillance-type rules, include only rules for this + // organisation's unit level. + // The organisation may not be configured for the data elements + // because they could be aggregated from a lower level. + if ( rule.getOrganisationUnitLevel() == sourceX.getLevel() ) + { + periodTypeRules.add( rule ); + } + } + } + + return periodTypeRules; + } + + /** + * Checks to see if the evaluation should go further for this + * evaluationRule, after the "current" data to evaluate has been fetched. + * For INTERACTIVE runs, we always go further (always return true.) For + * SCHEDULED runs, we go further only if something has changed since the + * last successful scheduled run -- either the rule definition or one of the + * "current" data element / option values on the left or right sides. + *

+ * For scheduled runs, remove all values for any attribute option combos + * where nothing has changed since the last run. + * + * @param lastUpdatedMapMap when each data value was last updated + * @param rule the rule that may be evaluated + * @return true if the rule should be evaluated with this data, false if not + */ + private boolean evaluateValidationCheck( MapMap currentValueMapMap, + MapMap lastUpdatedMapMap, ValidationRule rule ) + { + boolean evaluate = true; // Assume true for now. + + if ( ValidationRunType.SCHEDULED == context.getRunType() ) + { + if ( context.getLastScheduledRun() != null ) // True if no previous + // scheduled run + { + if ( rule.getLastUpdated().before( context.getLastScheduledRun() ) ) + { + // Get the "current" DataElementOperands from this rule: + // Left+Right sides for VALIDATION, Left side only for + // SURVEILLANCE. + Collection deos = context.getExpressionService() + .getOperandsInExpression( rule.getLeftSide().getExpression() ); + + if ( rule.getRuleType() == RuleType.VALIDATION ) + { + // Make a copy so we can add to it. + deos = new HashSet<>( deos ); + deos.addAll( context.getExpressionService() + .getOperandsInExpression( rule.getRightSide().getExpression() ) ); + } + + // Return true if any data is more recent than the last + // scheduled run, otherwise return false. + evaluate = false; + + for ( Map.Entry> entry : lastUpdatedMapMap.entrySet() ) + { + boolean saveThisCombo = false; + + for ( DataElementOperand deo : deos ) + { + Date lastUpdated = entry.getValue().get( deo ); + + if ( lastUpdated != null && lastUpdated.after( context.getLastScheduledRun() ) ) + { + saveThisCombo = true; // True if new/updated + // data. + evaluate = true; + break; + } + } + + if ( !saveThisCombo ) + { + currentValueMapMap.remove( entry.getKey() ); + } + } + } + } + } + return evaluate; + } + + /** + * Gets the data elements for which values should be fetched recursively if + * they are not collected for an organisation unit. + * + * @param rules ValidationRules to be evaluated + * @return the data elements to fetch recursively + */ + private Set getRecursiveCurrentDataElements( Set rules ) + { + Set recursiveCurrentDataElements = new HashSet<>(); + + for ( ValidationRule rule : rules ) + { + if ( rule.getRuleType() == RuleType.SURVEILLANCE && rule.getCurrentDataElements() != null ) + { + recursiveCurrentDataElements.addAll( rule.getCurrentDataElements() ); + } + } + + return recursiveCurrentDataElements; + } + + /** + * Evaluates an expression, returning a map of values by attribute option + * combo. + * + * @param expression expression to evaluate. + * @param valueMap Map of value maps, by attribute option combo. + * @param incompleteValuesMap map of values that were incomplete. + * @return map of values. + */ + private Map getExpressionValueMap( Expression expression, + MapMap valueMap, SetMap incompleteValuesMap ) + { + Map expressionValueMap = new HashMap<>(); + + for ( Map.Entry> entry : valueMap.entrySet() ) + { + Double value = context.getExpressionService().getExpressionValue( expression, entry.getValue(), + context.getConstantMap(), null, null, incompleteValuesMap.getSet( entry.getKey() ), null ); + + if ( MathUtils.isValidDouble( value ) ) + { + expressionValueMap.put( entry.getKey(), value ); + } + } + + return expressionValueMap; + } + + /** + * Gets data values for a given organisation unit and period, recursing if + * necessary to sum the values from child organisation units. + * + * @param periodTypeX period type which we are evaluating + * @param ruleDataElements data elements configured for the rule + * @param sourceDataElements data elements configured for the organisation + * unit + * @param recursiveDataElements data elements for which we will recurse if + * necessary + * @param allowedPeriodTypes all the periods in which we might find the data + * values + * @param period period in which we are looking for values + * @param source organisation unit for which we are looking for values + * @param lastUpdatedMap map showing when each data values was last updated + * @param incompleteValuesMap ongoing set showing which values were found + * but not from all children, mapped by attribute option combo. + * @return map of attribute option combo to map of values found. + */ + private MapMap getValueMap( PeriodTypeExtended periodTypeX, + Collection ruleDataElements, Collection sourceDataElements, + Set recursiveDataElements, Collection allowedPeriodTypes, Period period, + OrganisationUnit source, MapMap lastUpdatedMap, + SetMap incompleteValuesMap ) + { + Set dataElementsToGet = new HashSet<>( ruleDataElements ); + dataElementsToGet.retainAll( sourceDataElements ); + + log.trace( "getDataValueMapRecursive: source:" + source.getName() + " ruleDataElements[" + + ruleDataElements.size() + "] sourceDataElements[" + sourceDataElements.size() + "] elementsToGet[" + + dataElementsToGet.size() + "] recursiveDataElements[" + recursiveDataElements.size() + + "] allowedPeriodTypes[" + allowedPeriodTypes.size() + "]" ); + + MapMap dataValueMap = null; + + if ( dataElementsToGet.isEmpty() ) + { + // We still might get something recursively + dataValueMap = new MapMap<>(); + } + else + { + dataValueMap = context.getDataValueService().getDataValueMapByAttributeCombo( dataElementsToGet, + period.getStartDate(), source, allowedPeriodTypes, context.getAttributeCombo(), + context.getCogDimensionConstraints(), context.getCoDimensionConstraints(), lastUpdatedMap ); + } + + // See if there are any data elements we need to get recursively: + Set recursiveDataElementsNeeded = new HashSet<>( recursiveDataElements ); + recursiveDataElementsNeeded.removeAll( dataElementsToGet ); + + if ( !recursiveDataElementsNeeded.isEmpty() ) + { + int childCount = 0; + MapMap childValueCounts = new MapMap<>(); + + for ( OrganisationUnit child : source.getChildren() ) + { + Collection childDataElements = periodTypeX.getSourceDataElements().get( child ); + MapMap childMap = getValueMap( periodTypeX, + recursiveDataElementsNeeded, childDataElements, recursiveDataElementsNeeded, allowedPeriodTypes, + period, child, lastUpdatedMap, incompleteValuesMap ); + + for ( Map.Entry> entry : childMap.entrySet() ) + { + int combo = entry.getKey(); + + for ( Map.Entry e : entry.getValue().entrySet() ) + { + DataElementOperand deo = e.getKey(); + Double childValue = e.getValue(); + + Double baseValue = dataValueMap.getValue( combo, deo ); + dataValueMap.putEntry( combo, deo, baseValue == null ? childValue : baseValue + childValue ); + + Integer childValueCount = childValueCounts.getValue( combo, deo ); + childValueCounts.putEntry( combo, deo, childValueCount == null ? 1 : childValueCount + 1 ); + } + } + + childCount++; + } + + for ( Map.Entry> entry : childValueCounts.entrySet() ) + { + int combo = entry.getKey(); + + for ( Map.Entry e : entry.getValue().entrySet() ) + { + DataElementOperand deo = e.getKey(); + Integer childValueCount = e.getValue(); + + if ( childValueCount != childCount ) + { + // Remember that we found this DataElementOperand value + // in some but not all children + incompleteValuesMap.putValue( combo, deo ); + } + } + } + } + + return dataValueMap; + } + + /* Generalized surveillance rules */ + + /** + * Returns the right-side evaluated value of the validation rule. + * + * @param source organisation unit being evaluated + * @param periodTypeX period type being evaluated + * @param period period being evaluated + * @param rule ValidationRule being evaluated + * @param currentValueMap current values already fetched + * @param periodTypes applicable period types + * @param sourceDataElements the data elements collected by the organisation + * unit + * @return the right-side values, map by attribute category combo + */ + private ListMap getAggregateValueMap( Expression expression, OrganisationUnit source, + Period period, int window, int n_years, PeriodTypeExtended px, Collection periodTypes, + MapMap lastUpdatedMap, Collection sourceDataElements ) + { + ListMap results = new ListMap(); + CalendarPeriodType periodType = (CalendarPeriodType) period.getPeriodType(); + Calendar yearly = PeriodType.createCalendarInstance( period.getStartDate() ); + + for ( int years = 0; years <= n_years; years++ ) + { + // Defensive copy because createPeriod mutates Calendar. + Calendar each_year = PeriodType.createCalendarInstance( yearly.getTime() ); + // To track the period at the same time in preceding years. + Period base_period = periodType.createPeriod( each_year ); + + if ( years > 0 ) + { + // For past years, fetch a window around the period at the + // same time of year as this period. + gatherPeriodValues( results, expression, source, base_period, 0, px, periodTypes, lastUpdatedMap, + sourceDataElements ); + if ( window != 0 ) + gatherPeriodValues( results, expression, source, periodType.getNextPeriod( base_period ), + window - 1, px, periodTypes, lastUpdatedMap, sourceDataElements ); + } + if ( window != 0 ) + gatherPeriodValues( results, expression, source, periodType.getPreviousPeriod( base_period ), + 1 - window, px, periodTypes, lastUpdatedMap, sourceDataElements ); + + // Move to the previous year. + yearly.set( Calendar.YEAR, yearly.get( Calendar.YEAR ) - 1 ); + } + + return results; + } + + /** + * Gathers the values of an expression for a given organisation unit and + * period, accumulating a range of values around the given period. + *

+ * Note that for a surveillance-type rule, evaluating the right side + * expression can result in sampling multiple periods and/or child + * organisation units. + * + * @param results the ListMap into which results will be stored + * @param expression the expression to be evaluated + * @param source the organisation unit + * @param period the main period for the validation rule evaluation + * @param window how many periods (before and after) to collect + * @param px the period type extended information + * @param periodTypes the period types in which the data may exist + * @param sourceElements the data elements configured for this organisation + * unit + */ + private void gatherPeriodValues( ListMap results, Expression expression, OrganisationUnit source, + Period period, int window, PeriodTypeExtended px, Collection periodTypes, + MapMap lastUpdatedMap, Collection sourceElements ) + { + CalendarPeriodType periodType = (CalendarPeriodType) period.getPeriodType(); + Period periodInstance = context.getPeriodService().getPeriod( period.getStartDate(), period.getEndDate(), + periodType ); + if ( periodInstance == null ) + return; + Set dataElements = getExpressionDataElements( expression ); + SetMap incompleteValuesMap = new SetMap<>(); + MapMap dataValueMapByAttributeCombo = getValueMap( px, dataElements, + sourceElements, dataElements, periodTypes, periodInstance, source, lastUpdatedMap, incompleteValuesMap ); + Map eValues = getExpressionValueMap( expression, dataValueMapByAttributeCombo, + incompleteValuesMap ); + int direction = ((window < 0) ? (-1) : (window > 0) ? (1) : (0)); + int steps = ((direction > 0) ? (window) : (direction < 0) ? (-window) : (0)); + results.putValueMap( eValues ); + log.debug( "Gathering '" + expression.getExpression() + "' " + "at " + period + " (" + window + ") " + "from " + + source.getName() + " starting with:\n\t" + eValues ); + if ( direction == 0 ) + return; + Period scan = new Period( periodInstance ); + for ( int count = 0; count < steps; count++ ) + { + if ( direction < 0 ) + scan = periodType.getPreviousPeriod( scan ); + else + scan = periodType.getNextPeriod( scan ); + gatherPeriodValues( results, expression, source, scan, 0, px, periodTypes, lastUpdatedMap, sourceElements ); + } + } + + /** + * Returns the data elements referenced in an expression, as a set. + * + * This will return an empty set if e.getDataElementsInExpression returns + * null + * + * @param expression expression to evaluate. + * @return a Set of DataElement(s) + */ + private Set getExpressionDataElements( Expression e ) + { + Set elts = e.getDataElementsInExpression(); + if ( elts == null ) + return new HashSet(); + else + return elts; + } + + /** + * Evaluates an expression, returning a map of values by attribute option + * combo. + * + * @param expression expression to evaluate. + * @param valueMap Map of value maps, by attribute option combo. + * @param incompleteValuesMap map of values that were incomplete. + * @return map of values. + */ + private Map getRuleExpressionValueMap( Expression expression, + MapMap valueMap, SetMap incompleteValuesMap, + OrganisationUnit source, Period period, int window, int n_years, PeriodTypeExtended px, + Collection periodTypes, MapMap lastUpdatedMap, + Collection sourceElements ) + { + Map expressionValueMap = new HashMap<>(); + Map> aggregateValuesMap = new HashMap<>(); + Set aggregates = context.getExpressionService().getAggregatesInExpression( expression.getExpression() ); + + if ( aggregates.size() == 0 ) + return getExpressionValueMap( expression, valueMap, incompleteValuesMap ); + + for ( String subExpression : aggregates ) + { + Expression subexp = new Expression( subExpression, "aggregated", + new HashSet( sourceElements ) ); + ListMap aggregateValues = getAggregateValueMap( subexp, source, period, window, n_years, + px, periodTypes, lastUpdatedMap, sourceElements ); + for ( Integer attributeOptionCombo : aggregateValues.keySet() ) + { + ListMap aggmap; + if ( aggregateValuesMap.containsKey( attributeOptionCombo ) ) + aggmap = aggregateValuesMap.get( attributeOptionCombo ); + else + { + aggmap = new ListMap<>(); + aggregateValuesMap.put( attributeOptionCombo, aggmap ); + } + + aggmap.put( subExpression, aggregateValues.get( attributeOptionCombo ) ); + } + } + + for ( Map.Entry> entry : valueMap.entrySet() ) + { + Double value = context.getExpressionService().getExpressionValue( expression, entry.getValue(), + context.getConstantMap(), null, null, incompleteValuesMap.getSet( entry.getKey() ), + aggregateValuesMap.get( entry.getKey() ) ); + + if ( MathUtils.isValidDouble( value ) ) + { + expressionValueMap.put( entry.getKey(), value ); + } + } + + return expressionValueMap; + } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java' --- dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java 2016-01-08 19:08:02 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/expression/ExpressionServiceTest.java 2016-01-13 12:54:38 +0000 @@ -149,9 +149,9 @@ private String expressionG; private String expressionH; private String expressionI; + private String expressionK; private String expressionJ; - private String expressionJ0; - private String expressionJ1; + private String expressionL; private String descriptionA; private String descriptionB; @@ -264,9 +264,9 @@ expressionH = "#{" + deA.getUid() + SEPARATOR + coc.getUid() + "}*OUG{" + groupA.getUid() + "}"; expressionI = "#{" + opA.getDimensionItem() + "}*" + "#{" + deB.getDimensionItem() + "}+" + "C{" + constantA.getUid() + "}+5-" + "D{" + pdeA.getDimensionItem() + "}+" + "A{" + pteaA.getDimensionItem() + "}-10+" + "I{" + piA.getDimensionItem() + "}"; - expressionJ0 = "#{" + opA.getDimensionItem() + "}+#{" + opB.getDimensionItem() + "}"; - expressionJ = "1.5*AVG("+expressionJ0+")"; - expressionJ1 = "AVG("+expressionJ0+")+1.5*STDDEV("+expressionJ0+")"; + expressionJ = "#{" + opA.getDimensionItem() + "}+#{" + opB.getDimensionItem() + "}"; + expressionK = "1.5*AVG("+expressionJ+")"; + expressionL = "AVG("+expressionJ+")+1.5*STDDEV("+expressionJ+")"; descriptionA = "Expression A"; descriptionB = "Expression B"; @@ -358,27 +358,27 @@ @Test public void testGetAggregatesInExpression() { - Set dataElements = expressionService.getDataElementsInExpression( expressionJ ); - Set aggregates=expressionService.getAggregatesInExpression(expressionJ.toString()); + Set dataElements = expressionService.getDataElementsInExpression( expressionK ); + Set aggregates=expressionService.getAggregatesInExpression(expressionK.toString()); assertTrue( dataElements.size() == 2 ); assertTrue( dataElements.contains( deA ) ); assertTrue( dataElements.contains( deB ) ); assertEquals( 1, aggregates.size() ); - for (String subexp: aggregates) assertEquals(expressionJ0,subexp); - assertTrue( aggregates.contains( expressionJ0 ) ); + for (String subexp: aggregates) assertEquals(expressionJ,subexp); + assertTrue( aggregates.contains( expressionJ ) ); - dataElements=expressionService.getDataElementsInExpression( expressionJ ); - aggregates=expressionService.getAggregatesInExpression(expressionJ.toString()); + dataElements=expressionService.getDataElementsInExpression( expressionK ); + aggregates=expressionService.getAggregatesInExpression(expressionK.toString()); assertTrue( dataElements.size() == 2 ); assertTrue( dataElements.contains( deA ) ); assertTrue( dataElements.contains( deB ) ); assertEquals( 1, aggregates.size() ); - for (String subexp: aggregates) assertEquals(expressionJ0,subexp); - assertTrue( aggregates.contains( expressionJ0 ) ); + for (String subexp: aggregates) assertEquals(expressionJ,subexp); + assertTrue( aggregates.contains( expressionJ ) ); } @@ -496,8 +496,8 @@ assertTrue( expressionService.expressionIsValid( expressionD ).isValid() ); assertTrue( expressionService.expressionIsValid( expressionE ).isValid() ); assertTrue( expressionService.expressionIsValid( expressionH ).isValid() ); - assertTrue( expressionService.expressionIsValid( expressionJ ).isValid() ); - assertTrue( expressionService.expressionIsValid( expressionJ1 ).isValid() ); + assertTrue( expressionService.expressionIsValid( expressionK ).isValid() ); + assertTrue( expressionService.expressionIsValid( expressionL ).isValid() ); expressionA = "#{nonExisting" + SEPARATOR + coc.getUid() + "} + 12"; @@ -516,7 +516,7 @@ assertEquals( ExpressionValidationOutcome.EXPRESSION_IS_NOT_WELL_FORMED, expressionService.expressionIsValid( expressionA ) ); - expressionA=expressionJ.replace(")", ""); + expressionA=expressionK.replace(")", ""); assertEquals( ExpressionValidationOutcome.EXPRESSION_IS_NOT_WELL_FORMED, expressionService.expressionIsValid( expressionA ) ); === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/ArithmeticMean.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/ArithmeticMean.java 2016-01-06 17:14:10 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/ArithmeticMean.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,34 +36,48 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class ArithmeticMean extends PostfixMathCommand -implements PostfixMathCommandI +/** + * @author Kenneth Haase + */ +public class ArithmeticMean + extends PostfixMathCommand + implements PostfixMathCommandI { - public ArithmeticMean() { - numberOfParameters = 1; - } - - @Override - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - - // check the stack - checkStack(inStack); - - Object param= inStack.pop(); - if (param instanceof List) { - List vals=CustomFunctions.checkVector(param); - int n=vals.size(); - - if (n==0) { - inStack.push(new Double(0)); - } else { - double sum=0; for (Double v: vals) { - sum=sum+v;} - inStack.push(new Double(sum/n)); - } - } - else throw new ParseException("Invalid aggregate value in expression"); - } + public ArithmeticMean() + { + numberOfParameters = 1; + } + + @Override + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + if ( param instanceof List ) + { + List vals = CustomFunctions.checkVector( param ); + int n = vals.size(); + + if ( n == 0 ) + { + inStack.push( new Double( 0 ) ); + } + else + { + double sum = 0; + for ( Double v : vals ) + { + sum = sum + v; + } + inStack.push( new Double( sum / n ) ); + } + } + else + throw new ParseException( "Invalid aggregate value in expression" ); + } } - === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/Count.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/Count.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/Count.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,22 +36,28 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class Count extends PostfixMathCommand - implements PostfixMathCommandI +/** + * @author Kenneth Haase + */ +public class Count + extends PostfixMathCommand + implements PostfixMathCommandI { - public Count() { - numberOfParameters = 1; + public Count() + { + numberOfParameters = 1; } - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); - Object param= inStack.pop(); - List vals=CustomFunctions.checkVector(param); - inStack.push(vals.size()); + Object param = inStack.pop(); + List vals = CustomFunctions.checkVector( param ); + inStack.push( vals.size() ); } } - === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/CustomFunctions.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/CustomFunctions.java 2016-01-08 16:30:51 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/CustomFunctions.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,66 +39,93 @@ import org.nfunk.jep.ParseException; import org.nfunk.jep.function.PostfixMathCommandI; -public class CustomFunctions { - - private static Boolean init_done=false; - - public static Map aggregate_functions= - new HashMap(); - - public static void addFunctions(JEP parser) - { - if (!(init_done)) initCustomFunctions(); - for (Entry e: - aggregate_functions.entrySet()) { - String fname=e.getKey(); - PostfixMathCommandI cmd=e.getValue(); - parser.addFunction(fname,cmd); - } - } - - private static Pattern aggregate_prefix=Pattern.compile(""); - private static int n_aggregates=0; - public static Pattern getAggregatePrefixPattern(){ - if (!(init_done)) initCustomFunctions(); - if (n_aggregates==aggregate_functions.size()) - return aggregate_prefix; - else { - StringBuffer s=new StringBuffer(); int i=0; s.append("("); - for (String key: aggregate_functions.keySet()) { - if (i>0) s.append('|'); else i++; - s.append(key);} - s.append(")\\s*\\("); - aggregate_prefix=Pattern.compile(s.toString()); - n_aggregates=aggregate_functions.size(); - return aggregate_prefix;} - } - - public static void addAggregateFunction(String name,PostfixMathCommandI fn){ - aggregate_functions.put(name,fn); - } - - @SuppressWarnings("unchecked") - public static List checkVector(Object param) throws ParseException - { - if (param instanceof List) { - List vals=(List) param; - for (Object val: vals) { - if (!(val instanceof Double)) - throw new ParseException("Non numeric vector"); - } - return (List) param; - } - else throw new ParseException("Invalid vector argument"); - } - - private synchronized static void initCustomFunctions() { - if (init_done) return; else init_done=true; - CustomFunctions.addAggregateFunction("AVG",new ArithmeticMean()); - CustomFunctions.addAggregateFunction("STDDEV",new StandardDeviation()); - CustomFunctions.addAggregateFunction("MEDIAN",new MedianValue()); - CustomFunctions.addAggregateFunction("MAX",new MaxValue()); - CustomFunctions.addAggregateFunction("MIN",new MinValue()); - CustomFunctions.addAggregateFunction("COUNT",new Count()); - CustomFunctions.addAggregateFunction("VSUM",new VectorSum());} +/** + * @author Kenneth Haase + */ +public class CustomFunctions +{ + + private static Boolean init_done = false; + + public static Map aggregate_functions = new HashMap(); + + public static void addFunctions( JEP parser ) + { + if ( !(init_done) ) + initCustomFunctions(); + for ( Entry e : aggregate_functions.entrySet() ) + { + String fname = e.getKey(); + PostfixMathCommandI cmd = e.getValue(); + parser.addFunction( fname, cmd ); + } + } + + private static Pattern aggregate_prefix = Pattern.compile( "" ); + + private static int n_aggregates = 0; + + public static Pattern getAggregatePrefixPattern() + { + if ( !(init_done) ) + initCustomFunctions(); + if ( n_aggregates == aggregate_functions.size() ) + return aggregate_prefix; + else + { + StringBuffer s = new StringBuffer(); + int i = 0; + s.append( "(" ); + for ( String key : aggregate_functions.keySet() ) + { + if ( i > 0 ) + s.append( '|' ); + else + i++; + s.append( key ); + } + s.append( ")\\s*\\(" ); + aggregate_prefix = Pattern.compile( s.toString() ); + n_aggregates = aggregate_functions.size(); + return aggregate_prefix; + } + } + + public static void addAggregateFunction( String name, PostfixMathCommandI fn ) + { + aggregate_functions.put( name, fn ); + } + + @SuppressWarnings( "unchecked" ) + public static List checkVector( Object param ) + throws ParseException + { + if ( param instanceof List ) + { + List vals = (List) param; + for ( Object val : vals ) + { + if ( !(val instanceof Double) ) + throw new ParseException( "Non numeric vector" ); + } + return (List) param; + } + else + throw new ParseException( "Invalid vector argument" ); + } + + private synchronized static void initCustomFunctions() + { + if ( init_done ) + return; + else + init_done = true; + CustomFunctions.addAggregateFunction( "AVG", new ArithmeticMean() ); + CustomFunctions.addAggregateFunction( "STDDEV", new StandardDeviation() ); + CustomFunctions.addAggregateFunction( "MEDIAN", new MedianValue() ); + CustomFunctions.addAggregateFunction( "MAX", new MaxValue() ); + CustomFunctions.addAggregateFunction( "MIN", new MinValue() ); + CustomFunctions.addAggregateFunction( "COUNT", new Count() ); + CustomFunctions.addAggregateFunction( "VSUM", new VectorSum() ); + } } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MaxValue.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MaxValue.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MaxValue.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,26 +36,36 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class MaxValue extends PostfixMathCommand implements PostfixMathCommandI { - public MaxValue() { - numberOfParameters = 1; - } - - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); - - Object param = inStack.pop(); - List vals = CustomFunctions.checkVector(param); - Double max = null; - for (Double v : vals) { - if (max == null) - max = v; - else if (v > max) - max = v; - } - inStack.push(new Double(max)); - } +/** + * @author Kenneth Haase + */ +public class MaxValue + extends PostfixMathCommand + implements PostfixMathCommandI +{ + public MaxValue() + { + numberOfParameters = 1; + } + + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + List vals = CustomFunctions.checkVector( param ); + Double max = null; + for ( Double v : vals ) + { + if ( max == null ) + max = v; + else if ( v > max ) + max = v; + } + inStack.push( new Double( max ) ); + } } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MedianValue.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MedianValue.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MedianValue.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,24 +36,35 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class MedianValue extends PostfixMathCommand implements PostfixMathCommandI { - public MedianValue() { - numberOfParameters = 1; - } - - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); - - Object param = inStack.pop(); - List vals = CustomFunctions.checkVector(param); - int n = vals.size(); - // Sort it here - if (n % 2 == 0) { - inStack.push(new Double((vals.get(n / 2) + vals.get(n / 2 + 1)) / 2)); - } else - inStack.push(new Double(vals.get((n + 1) / 2))); - } +/** + * @author Kenneth Haase + */ +public class MedianValue + extends PostfixMathCommand + implements PostfixMathCommandI +{ + public MedianValue() + { + numberOfParameters = 1; + } + + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + List vals = CustomFunctions.checkVector( param ); + int n = vals.size(); + // Sort it here + if ( n % 2 == 0 ) + { + inStack.push( new Double( (vals.get( n / 2 ) + vals.get( n / 2 + 1 )) / 2 ) ); + } + else + inStack.push( new Double( vals.get( (n + 1) / 2 ) ) ); + } } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MinValue.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MinValue.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/MinValue.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,26 +36,36 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class MinValue extends PostfixMathCommand implements PostfixMathCommandI { - public MinValue() { - numberOfParameters = 1; - } - - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); - - Object param = inStack.pop(); - List vals = CustomFunctions.checkVector(param); - Double min = null; - for (Double v : vals) { - if (min == null) - min = v; - else if (v < min) - min = v; - } - inStack.push(new Double(min)); - } +/** + * @author Kenneth Haase + */ +public class MinValue + extends PostfixMathCommand + implements PostfixMathCommandI +{ + public MinValue() + { + numberOfParameters = 1; + } + + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + List vals = CustomFunctions.checkVector( param ); + Double min = null; + for ( Double v : vals ) + { + if ( min == null ) + min = v; + else if ( v < min ) + min = v; + } + inStack.push( new Double( min ) ); + } } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/StandardDeviation.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/StandardDeviation.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/StandardDeviation.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,33 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,34 +36,48 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class StandardDeviation extends PostfixMathCommand implements PostfixMathCommandI { - public StandardDeviation() { - numberOfParameters = 1; - } - - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); - - Object param = inStack.pop(); - List vals = CustomFunctions.checkVector(param); - int n = vals.size(); - if (n == 0) { - inStack.push(new Double(0)); - } else { - double sum = 0, sum2 = 0, mean, variance; - for (Double v : vals) { - sum = sum + v; - } - ; - mean = sum / n; - for (Double v : vals) { - sum2 = sum2 + ((v - mean) * (v - mean)); - } - variance = sum2 / n; - inStack.push(new Double(Math.sqrt(variance))); - } - } +/** + * @author Kenneth Haase + */ +public class StandardDeviation + extends PostfixMathCommand + implements PostfixMathCommandI +{ + public StandardDeviation() + { + numberOfParameters = 1; + } + + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + List vals = CustomFunctions.checkVector( param ); + int n = vals.size(); + if ( n == 0 ) + { + inStack.push( new Double( 0 ) ); + } + else + { + double sum = 0, sum2 = 0, mean, variance; + for ( Double v : vals ) + { + sum = sum + v; + } + + mean = sum / n; + for ( Double v : vals ) + { + sum2 = sum2 + ((v - mean) * (v - mean)); + } + variance = sum2 / n; + inStack.push( new Double( Math.sqrt( variance ) ) ); + } + } } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/VectorSum.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/VectorSum.java 2015-12-22 20:42:54 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/jep/VectorSum.java 2016-01-13 12:54:38 +0000 @@ -1,5 +1,36 @@ package org.hisp.dhis.system.jep; +/* + * Copyright (c) 2004-2016, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @author Kenneth Haase + */ import java.util.List; import java.util.Stack; import java.lang.Object; @@ -8,32 +39,43 @@ import org.nfunk.jep.function.PostfixMathCommand; import org.nfunk.jep.function.PostfixMathCommandI; -public class VectorSum extends PostfixMathCommand -implements PostfixMathCommandI +public class VectorSum + extends PostfixMathCommand + implements PostfixMathCommandI { - public VectorSum() { - numberOfParameters = 1; - } - - // nFunk's JEP run() method uses the raw Stack type - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void run(Stack inStack) throws ParseException { - // check the stack - checkStack(inStack); - - Object param= inStack.pop(); - if (param instanceof List) { - List vals=CustomFunctions.checkVector(param); - int n=vals.size(); - if (n==0) { - inStack.push(new Double(0)); - } else { - double sum=0; for (Double v: vals) { - sum=sum+v;} - inStack.push(new Double(sum)); - } - } - else throw new ParseException("Invalid aggregate value in expression"); - } + public VectorSum() + { + numberOfParameters = 1; + } + + // nFunk's JEP run() method uses the raw Stack type + @SuppressWarnings( { "rawtypes", "unchecked" } ) + public void run( Stack inStack ) + throws ParseException + { + // check the stack + checkStack( inStack ); + + Object param = inStack.pop(); + if ( param instanceof List ) + { + List vals = CustomFunctions.checkVector( param ); + int n = vals.size(); + if ( n == 0 ) + { + inStack.push( new Double( 0 ) ); + } + else + { + double sum = 0; + for ( Double v : vals ) + { + sum = sum + v; + } + inStack.push( new Double( sum ) ); + } + } + else + throw new ParseException( "Invalid aggregate value in expression" ); + } } -