=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2013-10-08 19:10:40 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2013-10-09 05:16:53 +0000 @@ -152,30 +152,36 @@ public int compareTo( ValidationResult other ) { - int result = source.getName().compareTo( other.source.getName() ); - - if ( result == 0 ) - { - result = period.getStartDate().compareTo( other.period.getStartDate() ); - - if ( result == 0 ) - { - result = period.getEndDate().compareTo( other.period.getEndDate() ); - - if ( result == 0 ) - { - result = validationImportanceOrder( validationRule.getImportance() ) - - validationImportanceOrder( other.validationRule.getImportance() ); - - if ( result == 0 ) - { - result = validationRule.getLeftSide().getDescription() - .compareTo( other.validationRule.getLeftSide().getDescription() ); - } - } - } - } - return result; + if ( source.getName().compareTo( other.source.getName() ) != 0 ) + { + return source.getName().compareTo( other.source.getName() ); + } + else if ( period.getStartDate().compareTo( other.period.getStartDate() ) != 0 ) + { + return period.getStartDate().compareTo( other.period.getStartDate() ); + } + else if ( source.getName().compareTo( other.source.getName() ) != 0 ) + { + return source.getName().compareTo( other.source.getName() ); + } + else if ( period.getStartDate().compareTo( other.period.getStartDate() ) != 0 ) + { + return period.getStartDate().compareTo( other.period.getStartDate() ); + } + else if ( period.getEndDate().compareTo( other.period.getEndDate() ) != 0 ) + { + return period.getEndDate().compareTo( other.period.getEndDate() ); + } + else if ( validationRule.getImportance().compareTo( other.validationRule.getImportance() ) != 0 ) + { + return validationImportanceOrder( validationRule.getImportance() ) + - validationImportanceOrder( other.validationRule.getImportance() ); + } + else + { + return validationRule.getLeftSide().getDescription() + .compareTo( other.validationRule.getLeftSide().getDescription() ); + } } private int validationImportanceOrder ( String importance ) === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java 2013-10-08 19:10:40 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java 2013-10-09 05:16:53 +0000 @@ -33,16 +33,10 @@ import static org.hisp.dhis.i18n.I18nUtils.getObjectsBetweenByName; import static org.hisp.dhis.i18n.I18nUtils.getObjectsByName; import static org.hisp.dhis.i18n.I18nUtils.i18n; -import static org.hisp.dhis.system.util.MathUtils.expressionIsTrue; -import static org.hisp.dhis.system.util.MathUtils.getRounded; -import static org.hisp.dhis.system.util.MathUtils.zeroIfNull; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -51,13 +45,10 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.GenericIdentifiableObjectStore; @@ -68,7 +59,6 @@ import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.expression.ExpressionService; -import org.hisp.dhis.expression.Operator; import org.hisp.dhis.i18n.I18nService; import org.hisp.dhis.message.MessageService; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -97,150 +87,6 @@ { private static final Log log = LogFactory.getLog( DefaultValidationRuleService.class ); - /** - * Defines how many decimal places for rounding the left and right side - * evaluation values in the report of results. - */ - private static final int DECIMALS = 1; - - /** - * Defines the types of alert run. - */ - private enum ValidationRunType - { - INTERACTIVE, ALERT - } - - /** - * This private subclass holds information for each organisation unit that - * is needed during a validation run (either interactive or an alert run). - * - * It is important that they should be copied from Hibernate lazy - * collections before the multithreaded part of the run starts, otherwise - * the threads may not be able to access these values. - */ - private class OrganisationUnitExtended - { - OrganisationUnit source; - - Collection children; - - int level; - - public String toString() - { - return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) - .append( "\n name", source.getName() ).append( "\n children[", children.size() + "]" ) - .append( "\n level", level ).toString(); - } - } - - /** - * This private subclass holds information for each period type that is - * needed during a validation run (either interactive or an alert run). - * - * By computing these values once at the start of a validation run, we avoid - * the overhead of having to compute them during the processing of every - * organisation unit. For some of these properties this is also important - * because they should be copied from Hibernate lazy collections before the - * multithreaded part of the run starts, otherwise the threads may not be - * able to access these values. - */ - private class PeriodTypeExtended - { - PeriodType periodType; - - Collection periods; - - Collection rules; - - Collection dataElements; - - Collection allowedPeriodTypes; - - Map> sourceDataElements; - - public String toString() - { - return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) - .append( "\n periodType", periodType ) - .append( "\n periods", (Arrays.toString( periods.toArray() )) ) - .append( "\n rules", (Arrays.toString( rules.toArray() )) ) - .append( "\n dataElements", (Arrays.toString( dataElements.toArray() )) ) - .append( "\n allowedPeriodTypes", (Arrays.toString( allowedPeriodTypes.toArray() )) ) - .append( "\n sourceDataElements", "[" + sourceDataElements.size() + "]" ).toString(); - } - } - - /** - * This private subclass holds common values that are used during a - * validation run (either interactive or an alert run.) These values don't - * change during the multi-threaded tasks (except that results entries are - * added in a threadsafe way.) - * - * Some of the values are precalculated collections, to save CPU time during - * the run. All of these values are stored in this single "context" object - * to allow a single object reference for each of the scheduled tasks. (This - * also reduces the amount of memory needed to queue all the multi-threaded - * tasks.) - * - * For some of these properties this is also important because they should - * be copied from Hibernate lazy collections before the multithreaded part - * of the run starts, otherwise the threads may not be able to access these - * values. - */ - private class ValidationRunContext - { - private Map PeriodTypeExtendedMap; - - private ValidationRunType runType; - - private Date lastAlertRun; - - private Map constantMap; - - private Collection sourceXs; - - private Collection validationResults; - - public String toString() - { - return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) - .append( "\n PeriodTypeExtendedMap", (Arrays.toString( PeriodTypeExtendedMap.entrySet().toArray() )) ) - .append( "\n runType", runType ).append( "\n lastAlertRun", lastAlertRun ) - .append( "\n constantMap", "[" + constantMap.size() + "]" ) - .append( "\n sourceXs", Arrays.toString( sourceXs.toArray() ) ) - .append( "\n validationResults", Arrays.toString( validationResults.toArray() ) ).toString(); - } - } - - /** - * Runs a validation task on a thread within a multi-threaded validation - * run. - * - * Each thread looks for validation results in a different organisation - * unit. - */ - private class ValidationWorkerThread - implements Runnable - { - private OrganisationUnitExtended sourceX; - - private ValidationRunContext context; - - private ValidationWorkerThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) - { - this.sourceX = sourceX; - this.context = context; - } - - @Override - public void run() - { - validateSource( sourceX, context ); - } - } - // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- @@ -433,8 +279,8 @@ } /** - * Evaluates validation rules for a collection of organisation units. This - * method breaks the job down by organisation unit. It assigns the + * Evaluates validation rules for a collection of organisation units. + * This method breaks the job down by organisation unit. It assigns the * evaluation for each organisation unit to a task that can be evaluated * independently in a multithreaded environment. * @@ -450,159 +296,39 @@ private Collection validateInternal( Collection sources, Collection periods, Collection rules, ValidationRunType runType, Date lastAlertRun ) { - ValidationRunContext context = buildNewContext( sources, periods, rules, ValidationRunType.ALERT, lastAlertRun ); - boolean singleThreadedOption = false; - - if ( singleThreadedOption ) - { - for ( OrganisationUnitExtended sourceX : context.sourceXs ) - { - validateSource( sourceX, context ); - } - } - else - { - int threadPoolSize = SystemUtils.getCpuCores(); - if ( threadPoolSize > 2 ) - { - threadPoolSize--; - } - if ( threadPoolSize > sources.size() ) - { - threadPoolSize = sources.size(); - } - - ExecutorService executor = Executors.newFixedThreadPool( threadPoolSize ); - - for ( OrganisationUnitExtended sourceX : context.sourceXs ) - { - Runnable worker = new ValidationWorkerThread( sourceX, context ); - executor.execute( worker ); - } - - executor.shutdown(); - try - { - executor.awaitTermination( 23, TimeUnit.HOURS ); - } - catch ( InterruptedException e ) - { - executor.shutdownNow(); - } - } - return context.validationResults; - } - - /** - * Creates and fills a new context object for a validation run. - * - * @param sources organisation units for validation - * @param periods periods for validation - * @param rules validation rules for validation - * @param runType whether this is an INTERACTIVE or ALERT run - * @param lastAlertRun (for ALERT runs) date of previous alert run - * @return context object for this run - */ - private ValidationRunContext buildNewContext( Collection sources, Collection periods, - Collection rules, ValidationRunType runType, Date lastAlertRun ) - { - ValidationRunContext context = new ValidationRunContext(); - context.runType = runType; - context.lastAlertRun = lastAlertRun; - context.validationResults = new ConcurrentLinkedQueue(); // thread-safe - context.PeriodTypeExtendedMap = new HashMap(); - context.sourceXs = new HashSet(); - - context.constantMap = new HashMap(); - context.constantMap.putAll( constantService.getConstantMap() ); - - // Group the periods by period type. - for ( Period period : periods ) - { - PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( context, period.getPeriodType() ); - periodTypeX.periods.add( period ); - } - - for ( ValidationRule rule : rules ) - { - // Find the period type extended for this rule - PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( context, rule.getPeriodType() ); - periodTypeX.rules.add( rule ); // Add this rule to the period type ext. - - if ( rule.getCurrentDataElements() != null ) - { - // Add this rule's data elements to the period extended. - periodTypeX.dataElements.addAll( rule.getCurrentDataElements() ); - } - // Add the allowed period types for this rule's data elements: - periodTypeX.allowedPeriodTypes.addAll( getAllowedPeriodTypesForDataElements( rule.getCurrentDataElements(), - rule.getPeriodType() ) ); - } - - // We only need to keep period types that are selected and also used by - // rules that are selected. - // Start by making a defensive copy so we can delete while iterating. - Set periodTypeXs = new HashSet( context.PeriodTypeExtendedMap.values() ); - for ( PeriodTypeExtended periodTypeX : periodTypeXs ) - { - if ( periodTypeX.periods.isEmpty() || periodTypeX.rules.isEmpty() ) - { - context.PeriodTypeExtendedMap.remove( periodTypeX.periodType ); - } - } - - for ( OrganisationUnit source : sources ) - { - OrganisationUnitExtended sourceX = new OrganisationUnitExtended(); - sourceX.source = source; - sourceX.children = new HashSet( source.getChildren() ); - sourceX.level = source.getOrganisationUnitLevel(); - context.sourceXs.add( sourceX ); - - Map> sourceDataElementsByPeriodType = source - .getDataElementsInDataSetsByPeriodType(); - for ( PeriodTypeExtended periodTypeX : context.PeriodTypeExtendedMap.values() ) - { - Collection sourceDataElements = sourceDataElementsByPeriodType - .get( periodTypeX.periodType ); - if ( sourceDataElements != null ) - { - periodTypeX.sourceDataElements.put( source, sourceDataElements ); - } - else - { - periodTypeX.sourceDataElements.put( source, new HashSet() ); - } - } - } - - return context; - } - - /** - * Gets the PeriodTypeExtended from the context object. If not found, - * creates a new PeriodTypeExtended object, puts it into the context object, - * and returns it. - * - * @param context validation run context - * @param periodType period type to search for - * @return period type extended from the context object - */ - PeriodTypeExtended getOrCreatePeriodTypeExtended( ValidationRunContext context, PeriodType periodType ) - { - PeriodTypeExtended periodTypeX = context.PeriodTypeExtendedMap.get( periodType ); - if ( periodTypeX == null ) - { - periodTypeX = new PeriodTypeExtended(); - periodTypeX.periodType = periodType; - periodTypeX.periods = new HashSet(); - periodTypeX.rules = new HashSet(); - periodTypeX.dataElements = new HashSet(); - periodTypeX.allowedPeriodTypes = new HashSet(); - periodTypeX.sourceDataElements = new HashMap>(); - context.PeriodTypeExtendedMap.put( periodType, periodTypeX ); - } - return periodTypeX; + ValidationRunContext context = ValidationRunContext.getNewValidationRunContext( sources, periods, rules, + constantService.getConstantMap(), ValidationRunType.ALERT, lastAlertRun, + expressionService, periodService, dataValueService); + + int threadPoolSize = SystemUtils.getCpuCores(); + + if ( threadPoolSize > 2 ) + { + threadPoolSize--; + } + if ( threadPoolSize > sources.size() ) + { + threadPoolSize = sources.size(); + } + + ExecutorService executor = Executors.newFixedThreadPool( threadPoolSize ); + + for ( OrganisationUnitExtended sourceX : context.getSourceXs() ) + { + Runnable worker = new ValidationWorkerThread( sourceX, context ); + executor.execute( worker ); + } + + executor.shutdown(); + try + { + executor.awaitTermination( 23, TimeUnit.HOURS ); + } + catch ( InterruptedException e ) + { + executor.shutdownNow(); + } + return context.getValidationResults(); } /** @@ -630,509 +356,20 @@ // are present in the database. for ( PeriodType periodType : rulePeriodTypes ) { - // This is a bit awkward. The current periodType object is of type - // periodType, but not of type CalendarPeriodType. In other words, - // ( periodType instanceof CalendarPeriodType ) returns false! - // In order to do periodType calendar math, we want a real - // "CalendarPeriodType" instance. - // TODO just cast to calendar period type - CalendarPeriodType calendarPeriodType = getCalendarPeriodType( periodType ); - if ( calendarPeriodType != null ) - { - Period currentPeriod = calendarPeriodType.createPeriod(); - Period previousPeriod = calendarPeriodType.getPreviousPeriod( currentPeriod ); - periods.addAll( periodService.getIntersectingPeriodsByPeriodType( periodType, - previousPeriod.getStartDate(), currentPeriod.getEndDate() ) ); - // Note: If the last successful daily run was more than one day - // ago, we might consider adding some additional periods of type - // DailyPeriodType so we don't miss any alerts. - } + CalendarPeriodType calendarPeriodType = ( CalendarPeriodType ) periodType; + Period currentPeriod = calendarPeriodType.createPeriod(); + Period previousPeriod = calendarPeriodType.getPreviousPeriod( currentPeriod ); + periods.addAll( periodService.getIntersectingPeriodsByPeriodType( periodType, + previousPeriod.getStartDate(), currentPeriod.getEndDate() ) ); + // Note: If the last successful daily run was more than one day + // ago, we might consider adding some additional periods of type + // DailyPeriodType so we don't miss any alerts. } return periods; } /** - * Evaluates validation rules for a single organisation unit. This is the - * central method in validation rule evaluation. - * - * @param sourceX extended object of the organisation unit in which to run - * the validation rules - * @param context the validation run context - */ - private void validateSource( OrganisationUnitExtended sourceX, ValidationRunContext context ) - { - if ( context.validationResults.size() < (ValidationRunType.INTERACTIVE == context.runType ? MAX_INTERACTIVE_VIOLATIONS - : MAX_ALERT_VIOLATIONS) ) - { - for ( PeriodTypeExtended periodTypeX : context.PeriodTypeExtendedMap.values() ) - { - Collection sourceDataElements = periodTypeX.sourceDataElements.get( sourceX.source ); - Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, context, - sourceDataElements ); - - if ( !rules.isEmpty() ) - { - Set recursiveCurrentDataElements = getRecursiveCurrentDataElements( rules ); - for ( Period period : periodTypeX.periods ) - { - Map lastUpdatedMap = new HashMap(); - Set incompleteValues = new HashSet(); - Map currentValueMap = getDataValueMapRecursive( periodTypeX, - periodTypeX.dataElements, sourceDataElements, recursiveCurrentDataElements, - periodTypeX.allowedPeriodTypes, period, sourceX.source, lastUpdatedMap, incompleteValues ); - log.trace( "currentValueMap[" + currentValueMap.size() + "]" ); - - for ( ValidationRule rule : rules ) - { - if ( evaluateCheck( lastUpdatedMap, rule, context ) ) - { - Double leftSide = expressionService.getExpressionValue( rule.getLeftSide(), - currentValueMap, context.constantMap, null, incompleteValues ); - - if ( leftSide != null || Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - Double rightSide = getRightSideValue( sourceX.source, periodTypeX, period, rule, - currentValueMap, sourceDataElements, context ); - - if ( rightSide != null || Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - boolean violation = false; - - if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) - { - violation = (leftSide != null && rightSide == null) - || (leftSide == null && rightSide != null); - } - else if ( leftSide != null && rightSide != null ) - { - violation = !expressionIsTrue( leftSide, rule.getOperator(), rightSide ); - } - - if ( violation ) - { - context.validationResults.add( new ValidationResult( period, - sourceX.source, rule, getRounded( zeroIfNull( leftSide ), DECIMALS ), - getRounded( zeroIfNull( rightSide ), DECIMALS ) ) ); - } - - log.trace( "-->Evaluated " + rule.getName() + ": " - + (violation ? "violation" : "OK") + " " + leftSide.toString() + " " - + rule.getOperator() + " " + rightSide.toString() + " (" - + context.validationResults.size() + " results)" ); - } - } - } - } - } - } - } - } - } - - /** - * Checks to see if the evaluation should go further for this - * evaluationRule, after the "current" data to evaluate has been fetched. - * For INTERACTIVE runs, we always go further (always return true.) For - * ALERT runs, we go further only if something has changed since the last - * successful alert run -- either the rule definition or one of the - * "current" data element / option values. - * - * @param lastUpdatedMap when each data value was last updated - * @param rule the rule that may be evaluated - * @param context the evaluation run context - * @return true if the rule should be evaluated with this data, false if not - */ - private boolean evaluateCheck( Map lastUpdatedMap, ValidationRule rule, - ValidationRunContext context ) - { - boolean evaluate = true; // Assume true for now. - - if ( ValidationRunType.ALERT == context.runType ) - { - if ( context.lastAlertRun != null ) // True if no previous alert run - { - if ( rule.getLastUpdated().before( context.lastAlertRun ) ) - { - // Get the "current" DataElementOperands from this rule: - // Left+Right sides for VALIDATION, Left side only for - // MONITORING - Collection deos = expressionService.getOperandsInExpression( rule.getLeftSide() - .getExpression() ); - if ( ValidationRule.RULE_TYPE_VALIDATION == rule.getRuleType() ) - { - // Make a copy so we can add to it. - deos = new HashSet( deos ); - - deos.addAll( expressionService.getOperandsInExpression( rule.getRightSide().getExpression() ) ); - } - - // Return true if any data is more recent than the last - // ALERT run, otherwise return false. - evaluate = false; - for ( DataElementOperand deo : deos ) - { - Date lastUpdated = lastUpdatedMap.get( deo ); - if ( lastUpdated != null && lastUpdated.after( context.lastAlertRun ) ) - { - evaluate = true; // True if new/updated data. - break; - } - } - } - } - } - return evaluate; - } - - /** - * Gets the rules that should be evaluated for a given organisation unit and - * period type. - * - * @param sourceX the organisation unit extended information - * @param periodTypeX the period type extended information - * @param context the alert run context - * @param sourceDataElements all data elements collected for this - * organisation unit - * @return - */ - private Set getRulesBySourceAndPeriodType( OrganisationUnitExtended sourceX, - PeriodTypeExtended periodTypeX, ValidationRunContext context, Collection sourceDataElements ) - { - Set periodTypeRules = new HashSet(); - - for ( ValidationRule rule : periodTypeX.rules ) - { - if ( (ValidationRule.RULE_TYPE_VALIDATION.equals( rule.getRuleType() )) ) - { - // For validation-type rules, include only rules where the - // organisation collects all the data elements in the rule. - // But if this is some funny kind of rule with no elements (like - // for testing), include it also. - Collection elements = rule.getCurrentDataElements(); - if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) ) - { - periodTypeRules.add( rule ); - } - } - else - { - // For monitoring-type rules, include only rules for this - // organisation's unit level. - // The organisation may not be configured for the data elements - // because they could be aggregated from a lower level. - if ( rule.getOrganisationUnitLevel() == sourceX.level ) - { - periodTypeRules.add( rule ); - } - } - } - - return periodTypeRules; - } - - /** - * Gets the data elements for which values should be fetched recursively if - * they are not collected for an organisation unit. - * - * @param rules ValidationRules to be evaluated - * @return the data elements to fetch recursively - */ - private Set getRecursiveCurrentDataElements( Set rules ) - { - Set recursiveCurrentDataElements = new HashSet(); - - for ( ValidationRule rule : rules ) - { - if ( ValidationRule.RULE_TYPE_MONITORING.equals( rule.getRuleType() ) - && rule.getCurrentDataElements() != null ) - { - recursiveCurrentDataElements.addAll( rule.getCurrentDataElements() ); - } - } - - return recursiveCurrentDataElements; - } - - /** - * Returns the right-side evaluated value of the validation rule. - * - * @param source organisation unit being evaluated - * @param periodTypeX period type being evaluated - * @param period period being evaluated - * @param rule ValidationRule being evaluated - * @param currentValueMap current values already fetched - * @param sourceDataElements the data elements collected by the organisation - * unit - * @param context the validation run context - * @return the right-side value - */ - private Double getRightSideValue( OrganisationUnit source, PeriodTypeExtended periodTypeX, Period period, - ValidationRule rule, Map currentValueMap, - Collection sourceDataElements, ValidationRunContext context ) - { - Double rightSideValue = null; - - // If ruleType is VALIDATION, the right side is evaluated using the same - // (current) data values. - // If ruleType is MONITORING but there are no data elements in the right - // side, then it doesn't matter - // what data values we use, so just supply the current data values in - // order to evaluate the (constant) expression. - - if ( ValidationRule.RULE_TYPE_VALIDATION.equals( rule.getRuleType() ) - || rule.getRightSide().getDataElementsInExpression().isEmpty() ) - { - rightSideValue = expressionService.getExpressionValue( rule.getRightSide(), currentValueMap, - context.constantMap, null ); - } - else - // ruleType equals MONITORING, and there are some data elements in the - // right side expression - { - CalendarPeriodType calendarPeriodType = getCalendarPeriodType( period.getPeriodType() ); - - if ( calendarPeriodType != null ) - { - Collection rightSidePeriodTypes = getAllowedPeriodTypesForDataElements( - rule.getPastDataElements(), rule.getPeriodType() ); - List sampleValues = new ArrayList(); - Calendar yearlyCalendar = PeriodType.createCalendarInstance( period.getStartDate() ); - int annualSampleCount = rule.getAnnualSampleCount() == null ? 0 : rule.getAnnualSampleCount(); - int sequentialSampleCount = rule.getSequentialSampleCount() == null ? 0 : rule - .getSequentialSampleCount(); - - for ( int annualCount = 0; annualCount <= annualSampleCount; annualCount++ ) - { - - // Defensive copy because createPeriod mutates Calendar. - Calendar calCopy = PeriodType.createCalendarInstance( yearlyCalendar.getTime() ); - - // To track the period at the same time in preceding years. - Period yearlyPeriod = calendarPeriodType.createPeriod( calCopy ); - - // For past years, fetch the period at the same time of year - // as this period, - // and any periods after this period within the - // sequentialPeriod limit. - // For the year of the stating period, we will only fetch - // previous sequential periods. - - if ( annualCount > 0 ) - { - // Fetch the period at the same time of year as the - // starting period. - evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, yearlyPeriod, - rule, sourceDataElements, context ); - - // Fetch the sequential periods after this prior-year - // period. - Period sequentialPeriod = new Period( yearlyPeriod ); - for ( int sequentialCount = 0; sequentialCount < sequentialSampleCount; sequentialCount++ ) - { - sequentialPeriod = calendarPeriodType.getNextPeriod( sequentialPeriod ); - evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, - sequentialPeriod, rule, sourceDataElements, context ); - } - } - - // Fetch the seqential periods before this period (both this - // year and past years): - Period sequentialPeriod = new Period( yearlyPeriod ); - for ( int sequentialCount = 0; sequentialCount < sequentialSampleCount; sequentialCount++ ) - { - sequentialPeriod = calendarPeriodType.getPreviousPeriod( sequentialPeriod ); - evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, - sequentialPeriod, rule, sourceDataElements, context ); - } - - // Move to the previous year: - yearlyCalendar.set( Calendar.YEAR, yearlyCalendar.get( Calendar.YEAR ) - 1 ); - } - - rightSideValue = rightSideAverage( rule, sampleValues, annualSampleCount, sequentialSampleCount ); - } - } - return rightSideValue; - } - - /** - * Evaluates the right side of a monitoring-type validation rule for a given - * organisation unit and period, and adds the value to a list of sample - * values. - * - * Note that for a monitoring-type rule, evaluating the right side - * expression can result in sampling multiple periods and/or child - * organisation units. - * - * @param periodTypeX the period type extended information - * @param sampleValues the list of sample values to add to - * @param source the organisation unit - * @param allowedPeriodTypes the period types in which the data may exist - * @param period the main period for the validation rule evaluation - * @param rule the monitoring-type rule being evaluated - * @param sourceDataElements the data elements configured for this - * organisation unit - * @param context the evaluation run context - */ - private void evaluateRightSidePeriod( PeriodTypeExtended periodTypeX, List sampleValues, - OrganisationUnit source, Collection allowedPeriodTypes, Period period, ValidationRule rule, - Collection sourceDataElements, ValidationRunContext context ) - { - Period periodInstance = periodService.getPeriod( period.getStartDate(), period.getEndDate(), - period.getPeriodType() ); - - if ( periodInstance != null ) - { - Set dataElements = rule.getRightSide().getDataElementsInExpression(); - Set incompleteValues = new HashSet(); - Map dataValueMap = getDataValueMapRecursive( periodTypeX, dataElements, - sourceDataElements, dataElements, allowedPeriodTypes, period, source, null, incompleteValues ); - Double value = expressionService.getExpressionValue( rule.getRightSide(), dataValueMap, - context.constantMap, null, incompleteValues ); - - if ( value != null ) - { - sampleValues.add( value ); - } - } - } - - /** - * Finds the average right-side sample value. This is used as the right-side - * expression value to evaluate a monitoring-type rule. - * - * @param rule monitoring-type rule being evaluated - * @param sampleValues sample values actually collected - * @param annualSampleCount number of annual samples tried for - * @param sequentialSampleCount number of sequential samples tried for - * @return - */ - Double rightSideAverage( ValidationRule rule, List sampleValues, int annualSampleCount, - int sequentialSampleCount ) - { - // Find the expected sample count for the last period of its type in the - // database: sequentialSampleCount for the immediately preceding periods - // in this year and for every past year: one sample for the same period - // in that year, plus sequentialSampleCounts before and after. - Double average = null; - if ( !sampleValues.isEmpty() ) - { - int expectedSampleCount = sequentialSampleCount + annualSampleCount * (1 + 2 * sequentialSampleCount); - int highOutliers = rule.getHighOutliers() == null ? 0 : rule.getHighOutliers(); - int lowOutliers = rule.getLowOutliers() == null ? 0 : rule.getLowOutliers(); - - // If we had fewer than the expected number of samples, then scale - // back - if ( highOutliers + lowOutliers > sampleValues.size() ) - { - highOutliers = (highOutliers * sampleValues.size()) / expectedSampleCount; - lowOutliers = (lowOutliers * sampleValues.size()) / expectedSampleCount; - } - - // If we (still) have any high and/or low outliers to remove, then - // sort the sample values and remove the high and/or low outliers. - if ( highOutliers + lowOutliers > 0 ) - { - Collections.sort( sampleValues ); - log.trace( "Removing " + highOutliers + " high and " + lowOutliers + " low outliers from " - + Arrays.toString( sampleValues.toArray() ) ); - sampleValues = sampleValues.subList( lowOutliers, sampleValues.size() - highOutliers ); - log.trace( "Result: " + Arrays.toString( sampleValues.toArray() ) ); - } - Double sum = 0.0; - for ( Double sample : sampleValues ) - { - sum += sample; - } - average = sum / sampleValues.size(); - } - return average; - } - - /** - * Gets data values for a given organisation unit and period, recursing if - * necessary to sum the values from child organisation units. - * - * @param periodTypeX period type which we are evaluating - * @param ruleDataElements data elements configured for the rule - * @param sourceDataElements data elements configured for the organisation - * unit - * @param recursiveDataElements data elements for which we will recurse if - * necessary - * @param allowedPeriodTypes all the periods in which we might find the data - * values - * @param period period in which we are looking for values - * @param source organisation unit for which we are looking for values - * @param lastUpdatedMap map showing when each data values was last updated - * @param incompleteValues ongoing list showing which values were found but - * not from all children - * @return the map of values found - */ - private Map getDataValueMapRecursive( PeriodTypeExtended periodTypeX, - Collection ruleDataElements, Collection sourceDataElements, - Set recursiveDataElements, Collection allowedPeriodTypes, Period period, - OrganisationUnit source, Map lastUpdatedMap, Set incompleteValues ) - { - Set dataElementsToGet = new HashSet( ruleDataElements ); - dataElementsToGet.retainAll( sourceDataElements ); - log.trace( "getDataValueMapRecursive: source:" + source.getName() + " elementsToGet[" - + dataElementsToGet.size() + "] allowedPeriodTypes[" + allowedPeriodTypes.size() + "]" ); - - Map dataValueMap; - - if ( dataElementsToGet.isEmpty() ) - { - // We still might get something recursively - dataValueMap = new HashMap(); - } - else - { - dataValueMap = dataValueService.getDataValueMap( dataElementsToGet, period.getStartDate(), source, - allowedPeriodTypes, lastUpdatedMap ); - } - - // See if there are any data elements we need to get recursively: - Set recursiveDataElementsNeeded = new HashSet( recursiveDataElements ); - recursiveDataElementsNeeded.removeAll( dataElementsToGet ); - if ( !recursiveDataElementsNeeded.isEmpty() ) - { - int childCount = 0; - Map childValueCounts = new HashMap(); - - for ( OrganisationUnit child : source.getChildren() ) - { - Collection childDataElements = periodTypeX.sourceDataElements.get( child ); - Map childMap = getDataValueMapRecursive( periodTypeX, - recursiveDataElementsNeeded, childDataElements, recursiveDataElementsNeeded, allowedPeriodTypes, - period, child, lastUpdatedMap, incompleteValues ); - - for ( DataElementOperand deo : childMap.keySet() ) - { - Double baseValue = dataValueMap.get( deo ); - dataValueMap.put( deo, baseValue == null ? childMap.get( deo ) : baseValue + childMap.get( deo ) ); - - Integer childValueCount = childValueCounts.get( deo ); - childValueCounts.put( deo, childValueCount == null ? 1 : childValueCount + 1 ); - } - - childCount++; - } - - for ( Map.Entry entry : childValueCounts.entrySet() ) - { - if ( childCount != entry.getValue() ) - { - // Found this DataElementOperand value in some but not all children - incompleteValues.add( entry.getKey() ); - } - } - } - - return dataValueMap; - } - - /** * Returns all validation-type rules which have specified data elements * assigned to them. * @@ -1185,10 +422,10 @@ if ( rule.getRuleType().equals( ValidationRule.RULE_TYPE_VALIDATION ) ) { validationRuleOperands.clear(); - validationRuleOperands.addAll( expressionService.getOperandsInExpression( rule.getLeftSide() - .getExpression() ) ); - validationRuleOperands.addAll( expressionService.getOperandsInExpression( rule.getRightSide() - .getExpression() ) ); + validationRuleOperands.addAll( expressionService.getOperandsInExpression( + rule.getLeftSide().getExpression() ) ); + validationRuleOperands.addAll( expressionService.getOperandsInExpression( + rule.getRightSide().getExpression() ) ); if ( operands.containsAll( validationRuleOperands ) ) { @@ -1201,64 +438,6 @@ } /** - * Finds all period types that may contain given data elements, whose period - * type interval is at least as long as the given period type. - * - * @param dataElements data elements to look for - * @param periodType the minimum-length period type - * @return all period types that are allowed for these data elements - */ - private Collection getAllowedPeriodTypesForDataElements( Collection dataElements, - PeriodType periodType ) - { - Collection allowedPeriodTypes = new HashSet(); - for ( DataElement dataElement : dataElements ) - { - for ( DataSet dataSet : dataElement.getDataSets() ) - { - if ( dataSet.getPeriodType().getFrequencyOrder() >= periodType.getFrequencyOrder() ) - { - allowedPeriodTypes.add( dataSet.getPeriodType() ); - } - } - } - return allowedPeriodTypes; - } - - /** - * Returns an instance of type CalendarPeriodType that matches the specified - * periodType. This can be needed in order to access the calendar-computing - * methods that are available in a CalendarPeriodType but not an ordinary - * PeriodType. - * - * Note: Perhaps this should be moved to PeriodService. Or perhaps some - * refactoring can be done in the relationship between PeriodType and - * CalendarPeriodType. - * - * @param periodType the period type of interest - * @return the corresponding CalendarPeriodType - */ - private CalendarPeriodType getCalendarPeriodType( PeriodType periodType ) - { - for ( PeriodType p : PeriodType.PERIOD_TYPES ) - { - if ( periodType.getName().equals( p.getName() ) ) - { - if ( p instanceof CalendarPeriodType ) - { - return (CalendarPeriodType) p; - } - else - { - log.error( "DefaultValidationRuleService.getCalendarPeriodType() - PeriodType.PERIOD_TYPES [" - + p.getName() + "] is not a CalendarPeriodType!" ); - } - } - } - return null; - } - - /** * At the end of an ALERT run, post messages to the users who want to see * the results. * @@ -1295,17 +474,14 @@ } // We will create one message for each set of users who receive the same - // subset - // of results. (Not necessarily the same as the set of users who receive - // alerts - // from the same subset of validation rules -- because some of these - // rules - // may return no results.) This saves on message storage space. + // subset of results. (Not necessarily the same as the set of users who + // receive alerts from the same subset of validation rules -- because + // some of these rules may return no results.) This saves on message + // storage space. // TODO: Encapsulate this in another level of Map by the user's - // language, and - // generate a message for each combination of ( target language, set of - // results ) + // language, and generate a message for each combination of ( target + // language, set of results ) Map, Set> messageMap = new HashMap, Set>(); for ( User user : userRulesMap.keySet() ) @@ -1385,33 +561,47 @@ + (importanceCountMap.get( "low" ) == null ? 0 : importanceCountMap.get( "low" )); // Construct the text of the message. - messageBuilder.append( "\n" ).append( "\n" ) + messageBuilder + .append( "\n" ) + .append( "\n" ) .append( "\n" ) .append( "\n" ) .append( subject ) .append( "\n" ) - // Repeat the subject line at the start of the message. - .append( "
\n" ).append( "\n" ).append( " \n" ).append( " \n" ) - .append( " \n" ).append( " \n" ) - .append( " \n" ).append( " \n" ) - .append( " \n" ).append( " \n" ) - .append( " \n" ).append( " \n" ); + .append( "
\n" ) + .append( "
Organisation UnitPeriodImportanceLeft side descriptionValueOperatorValueRight side description
\n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ) + .append( " \n" ); for ( ValidationResult result : results ) { ValidationRule rule = result.getValidationRule(); - messageBuilder.append( " \n" ).append( " \n" ); + messageBuilder + .append( " \n" ) + .append( " \n" ); } - messageBuilder.append( "
Organisation UnitPeriodImportanceLeft side descriptionValueOperatorValueRight side description
" ).append( result.getSource().getName() ) - .append( "<\td>\n" ).append( " " ).append( result.getPeriod().getName() ).append( "<\td>\n" ) - .append( " " ).append( rule.getImportance() ).append( "<\td>\n" ).append( " " ) - .append( rule.getLeftSide().getDescription() ).append( "<\td>\n" ).append( " " ) - .append( result.getLeftsideValue() ).append( "<\td>\n" ).append( " " ) - .append( rule.getOperator().toString() ).append( "<\td>\n" ).append( " " ) - .append( result.getRightsideValue() ).append( "<\td>\n" ).append( " " ) - .append( rule.getRightSide().getDescription() ).append( "<\td>\n" ).append( "
" ).append( result.getSource().getName() ).append( "<\td>\n" ) + .append( " " ).append( result.getPeriod().getName() ).append( "<\td>\n" ) + .append( " " ).append( rule.getImportance() ).append( "<\td>\n" ) + .append( " " ).append( rule.getLeftSide().getDescription() ).append( "<\td>\n" ) + .append( " " ).append( result.getLeftsideValue() ).append( "<\td>\n" ) + .append( " " ).append( rule.getOperator().toString() ).append( "<\td>\n" ) + .append( " " ).append( result.getRightsideValue() ).append( "<\td>\n" ) + .append( " " ).append( rule.getRightSide().getDescription() ).append( "<\td>\n" ) + .append( "
\n" ).append( "\n" ).append( "\n" ); + messageBuilder + .append( "\n" ) + .append( "\n" ) + .append( "\n" ); String messageText = messageBuilder.toString(); === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,86 @@ +package org.hisp.dhis.validation; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; +import java.util.HashSet; + +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.hisp.dhis.organisationunit.OrganisationUnit; + +/** + * Holds information for each organisation unit that is needed during + * a validation run (either interactive or an alert run). + * + * It is important that they should be copied from Hibernate lazy + * collections before the multithreaded part of the run starts, otherwise + * the threads may not be able to access these values. + * + * @author Jim Grace + */ +public class OrganisationUnitExtended +{ + private OrganisationUnit source; + + private Collection children; + + private int level; + + public OrganisationUnitExtended( OrganisationUnit source ) + { + this.source = source; + children = new HashSet( source.getChildren() ); + level = source.getOrganisationUnitLevel(); + } + + public String toString() + { + return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) + .append( "\n name", source.getName() ).append( "\n children[", children.size() + "]" ) + .append( "\n level", level ).toString(); + } + + // ------------------------------------------------------------------------- + // Set and get methods + // ------------------------------------------------------------------------- + + public OrganisationUnit getSource() { + return source; + } + + public Collection getChildren() { + return children; + } + + public int getLevel() { + return level; + } +} + === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,140 @@ +package org.hisp.dhis.validation; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodType; + +/** + * Holds information for each period type that is needed during + * a validation run (either interactive or an alert run). + * + * By computing these values once at the start of a validation run, we avoid + * the overhead of having to compute them during the processing of every + * organisation unit. For some of these properties this is also important + * because they should be copied from Hibernate lazy collections before the + * multithreaded part of the run starts, otherwise the threads may not be + * able to access these values. + * + * @author Jim Grace + */ +public class PeriodTypeExtended { + + private PeriodType periodType; + + private Collection periods; + + private Collection rules; + + private Collection dataElements; + + private Collection allowedPeriodTypes; + + private Map> sourceDataElements; + + public PeriodTypeExtended( PeriodType periodType ) + { + this.periodType = periodType; + periods = new HashSet(); + rules = new HashSet(); + dataElements = new HashSet(); + allowedPeriodTypes = new HashSet(); + sourceDataElements = new HashMap>(); + } + + public String toString() + { + return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) + .append( "\n periodType", periodType ) + .append( "\n periods", (Arrays.toString( periods.toArray() )) ) + .append( "\n rules", (Arrays.toString( rules.toArray() )) ) + .append( "\n dataElements", (Arrays.toString( dataElements.toArray() )) ) + .append( "\n allowedPeriodTypes", (Arrays.toString( allowedPeriodTypes.toArray() )) ) + .append( "\n sourceDataElements", "[" + sourceDataElements.size() + "]" ).toString(); + } + + // ------------------------------------------------------------------------- + // Set and get methods + // ------------------------------------------------------------------------- + + public PeriodType getPeriodType() { + return periodType; + } + + public Collection getPeriods() { + return periods; + } + + public void setPeriods(Collection periods) { + this.periods = periods; + } + + public Collection getRules() { + return rules; + } + + public void setRules(Collection rules) { + this.rules = rules; + } + + public Collection getDataElements() { + return dataElements; + } + + public void setDataElements(Collection dataElements) { + this.dataElements = dataElements; + } + + public Collection getAllowedPeriodTypes() { + return allowedPeriodTypes; + } + + public void setAllowedPeriodTypes(Collection allowedPeriodTypes) { + this.allowedPeriodTypes = allowedPeriodTypes; + } + + public Map> getSourceDataElements() { + return sourceDataElements; + } + + public void setSourceDataElements( + Map> sourceDataElements) { + this.sourceDataElements = sourceDataElements; + } +} === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRuleExtended.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRuleExtended.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRuleExtended.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,71 @@ +package org.hisp.dhis.validation; + +import java.util.Collection; + +import org.hisp.dhis.period.PeriodType; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Holds information for each validation rule that is needed during + * a validation run (either interactive or an alert run). + * + * By computing these values once at the start of a validation run, we avoid + * the overhead of having to compute them during the processing of every + * organisation unit. For some of these properties this is also important + * because they should be copied from Hibernate lazy collections before the + * multithreaded part of the run starts, otherwise the threads may not be + * able to access these values. + * + * @author Jim Grace + */ +public class ValidationRuleExtended { + + private ValidationRule rule; + + private Collection allowedPastPeriodTypes; + + public ValidationRuleExtended( ValidationRule rule, Collection allowedPastPeriodTypes ) + { + this.rule = rule; + this.allowedPastPeriodTypes = allowedPastPeriodTypes; + } + + // ------------------------------------------------------------------------- + // Set and get methods + // ------------------------------------------------------------------------- + + public ValidationRule getRule() { + return rule; + } + + public Collection getAllowedPastPeriodTypes() { + return allowedPastPeriodTypes; + } +} === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,300 @@ +package org.hisp.dhis.validation; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.datavalue.DataValueService; +import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; + +/** + * Holds common values that are used during a validation run (either interactive + * or an alert run.) These values don't change during the multi-threaded tasks + * (except that results entries are added in a threadsafe way.) + * + * Some of the values are precalculated collections, to save CPU time during + * the run. All of these values are stored in this single "context" object + * to allow a single object reference for each of the scheduled tasks. (This + * also reduces the amount of memory needed to queue all the multi-threaded + * tasks.) + * + * For some of these properties this is also important because they should + * be copied from Hibernate lazy collections before the multithreaded part + * of the run starts, otherwise the threads may not be able to access these + * values. + * + * @author Jim Grace + */ +public class ValidationRunContext +{ + private Map periodTypeExtendedMap; + + private ValidationRunType runType; + + private Date lastAlertRun; + + private Map constantMap; + + private Map ruleXMap; + + private Collection sourceXs; + + private Collection validationResults; + + private ExpressionService expressionService; + + private PeriodService periodService; + + private DataValueService dataValueService; + + private ValidationRunContext() + { + } + + public String toString() + { + return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) + .append( "\n PeriodTypeExtendedMap", (Arrays.toString( periodTypeExtendedMap.entrySet().toArray() )) ) + .append( "\n runType", runType ).append( "\n lastAlertRun", lastAlertRun ) + .append( "\n constantMap", "[" + constantMap.size() + "]" ) + .append( "\n ruleXMap", "[" + ruleXMap.size() + "]" ) + .append( "\n sourceXs", Arrays.toString( sourceXs.toArray() ) ) + .append( "\n validationResults", Arrays.toString( validationResults.toArray() ) ).toString(); + } + + /** + * Creates and fills a new context object for a validation run. + * + * @param sources organisation units for validation + * @param periods periods for validation + * @param rules validation rules for validation + * @param runType whether this is an INTERACTIVE or ALERT run + * @param lastAlertRun (for ALERT runs) date of previous alert run + * @return context object for this run + */ + public static ValidationRunContext getNewValidationRunContext( Collection sources, Collection periods, + Collection rules, Map constantMap, ValidationRunType runType, Date lastAlertRun, + ExpressionService expressionService, PeriodService periodService, DataValueService dataValueService ) + { + ValidationRunContext context = new ValidationRunContext(); + context.runType = runType; + context.lastAlertRun = lastAlertRun; + context.validationResults = new ConcurrentLinkedQueue(); // thread-safe + context.periodTypeExtendedMap = new HashMap(); + context.ruleXMap = new HashMap(); + context.sourceXs = new HashSet(); + context.constantMap = constantMap; + context.expressionService = expressionService; + context.periodService = periodService; + context.dataValueService = dataValueService; + context.initialize( sources, periods, rules ); + return context; + } + + /** + * Initializes context values based on sources, periods and rules + * + * @param sources + * @param periods + * @param rules + */ + private void initialize( Collection sources, Collection periods, Collection rules ) + { + // Group the periods by period type. + for ( Period period : periods ) + { + PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( period.getPeriodType() ); + periodTypeX.getPeriods().add( period ); + } + + for ( ValidationRule rule : rules ) + { + // Find the period type extended for this rule + PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( rule.getPeriodType() ); + periodTypeX.getRules().add( rule ); // Add this rule to the period type ext. + + if ( rule.getCurrentDataElements() != null ) + { + // Add this rule's data elements to the period extended. + periodTypeX.getDataElements().addAll( rule.getCurrentDataElements() ); + } + // Add the allowed period types for rule's current data elements: + periodTypeX.getAllowedPeriodTypes().addAll( + getAllowedPeriodTypesForDataElements( rule.getCurrentDataElements(), rule.getPeriodType() ) ); + + // Add the ValidationRuleExtended + Collection allowedPastPeriodTypes = + getAllowedPeriodTypesForDataElements( rule.getPastDataElements(), rule.getPeriodType() ); + ValidationRuleExtended ruleX = new ValidationRuleExtended( rule, allowedPastPeriodTypes ); + ruleXMap.put( rule, ruleX ); + } + + // We only need to keep period types that are selected and also used by + // rules that are selected. + // Start by making a defensive copy so we can delete while iterating. + Set periodTypeXs = new HashSet( periodTypeExtendedMap.values() ); + for ( PeriodTypeExtended periodTypeX : periodTypeXs ) + { + if ( periodTypeX.getPeriods().isEmpty() || periodTypeX.getRules().isEmpty() ) + { + periodTypeExtendedMap.remove( periodTypeX.getPeriodType() ); + } + } + + for ( OrganisationUnit source : sources ) + { + OrganisationUnitExtended sourceX = new OrganisationUnitExtended( source ); + sourceXs.add( sourceX ); + + Map> sourceDataElementsByPeriodType = source + .getDataElementsInDataSetsByPeriodType(); + for ( PeriodTypeExtended periodTypeX : periodTypeExtendedMap.values() ) + { + Collection sourceDataElements = sourceDataElementsByPeriodType + .get( periodTypeX.getPeriodType() ); + if ( sourceDataElements != null ) + { + periodTypeX.getSourceDataElements().put( source, sourceDataElements ); + } + else + { + periodTypeX.getSourceDataElements().put( source, new HashSet() ); + } + } + } + } + + /** + * Gets the PeriodTypeExtended from the context object. If not found, + * creates a new PeriodTypeExtended object, puts it into the context object, + * and returns it. + * + * @param context validation run context + * @param periodType period type to search for + * @return period type extended from the context object + */ + private PeriodTypeExtended getOrCreatePeriodTypeExtended( PeriodType periodType ) + { + PeriodTypeExtended periodTypeX = periodTypeExtendedMap.get( periodType ); + if ( periodTypeX == null ) + { + periodTypeX = new PeriodTypeExtended( periodType ); + periodTypeExtendedMap.put( periodType, periodTypeX ); + } + return periodTypeX; + } + + /** + * Finds all period types that may contain given data elements, whose period + * type interval is at least as long as the given period type. + * + * @param dataElements data elements to look for + * @param periodType the minimum-length period type + * @return all period types that are allowed for these data elements + */ + private static Collection getAllowedPeriodTypesForDataElements( Collection dataElements, + PeriodType periodType ) + { + Collection allowedPeriodTypes = new HashSet(); + if ( dataElements != null ) + { + for ( DataElement dataElement : dataElements ) + { + for ( DataSet dataSet : dataElement.getDataSets() ) + { + if ( dataSet.getPeriodType().getFrequencyOrder() >= periodType.getFrequencyOrder() ) + { + allowedPeriodTypes.add( dataSet.getPeriodType() ); + } + } + } + } + return allowedPeriodTypes; + } + + // ------------------------------------------------------------------------- + // Set and get methods + // ------------------------------------------------------------------------- + + public Map getPeriodTypeExtendedMap() { + return periodTypeExtendedMap; + } + + public ValidationRunType getRunType() { + return runType; + } + + public Date getLastAlertRun() { + return lastAlertRun; + } + + public Map getConstantMap() { + return constantMap; + } + + public Map getRuleXMap() { + return ruleXMap; + } + + public Collection getSourceXs() { + return sourceXs; + } + + public Collection getValidationResults() { + return validationResults; + } + + public ExpressionService getExpressionService() { + return expressionService; + } + + public PeriodService getPeriodService() { + return periodService; + } + + public DataValueService getDataValueService() { + return dataValueService; + } +} + === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunType.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunType.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunType.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,37 @@ +package org.hisp.dhis.validation; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Defines the types of alert run. + */ +public enum ValidationRunType +{ + INTERACTIVE, ALERT +} === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationWorkerThread.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationWorkerThread.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationWorkerThread.java 2013-10-09 05:16:53 +0000 @@ -0,0 +1,567 @@ +package org.hisp.dhis.validation; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.system.util.MathUtils.expressionIsTrue; +import static org.hisp.dhis.system.util.MathUtils.getRounded; +import static org.hisp.dhis.system.util.MathUtils.zeroIfNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataelement.DataElementOperand; +import org.hisp.dhis.expression.Operator; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.period.CalendarPeriodType; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodType; + +/** + * Runs a validation task on a thread within a multi-threaded validation run. + * + * Each thread looks for validation results in a different organisation unit. + * + * @author Jim Grace + */ +public class ValidationWorkerThread + implements Runnable +{ + private static final Log log = LogFactory.getLog( ValidationWorkerThread.class ); + + /** + * Defines how many decimal places for rounding the left and right side + * evaluation values in the report of results. + */ + private static final int DECIMALS = 1; + + private OrganisationUnitExtended sourceX; + + private ValidationRunContext context; + + public ValidationWorkerThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) + { + this.sourceX = sourceX; + this.context = context; + } + + @Override + public void run() + { + validateSource( sourceX, context ); + } + + /** + * Evaluates validation rules for a single organisation unit. This is the + * central method in validation rule evaluation. + * + * @param sourceX extended object of the organisation unit in which to run + * the validation rules + * @param context the validation run context + */ + private void validateSource( OrganisationUnitExtended sourceX, ValidationRunContext context ) + { + if ( context.getValidationResults().size() < ( ValidationRunType.INTERACTIVE == context.getRunType() ? + ValidationRuleService.MAX_INTERACTIVE_VIOLATIONS : ValidationRuleService.MAX_ALERT_VIOLATIONS) ) + { + for ( PeriodTypeExtended periodTypeX : context.getPeriodTypeExtendedMap().values() ) + { + Collection sourceDataElements = periodTypeX.getSourceDataElements().get( sourceX.getSource() ); + Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, context, + sourceDataElements ); + + if ( !rules.isEmpty() ) + { + Set recursiveCurrentDataElements = getRecursiveCurrentDataElements( rules ); + for ( Period period : periodTypeX.getPeriods() ) + { + Map lastUpdatedMap = new HashMap(); + Set incompleteValues = new HashSet(); + Map currentValueMap = getDataValueMapRecursive( periodTypeX, + periodTypeX.getDataElements(), sourceDataElements, recursiveCurrentDataElements, + periodTypeX.getAllowedPeriodTypes(), period, sourceX.getSource(), lastUpdatedMap, incompleteValues ); + log.trace( "currentValueMap[" + currentValueMap.size() + "]" ); + + for ( ValidationRule rule : rules ) + { + CalendarPeriodType calendarPeriodType = ( CalendarPeriodType ) rule.getPeriodType(); //TODO: remove + + if ( evaluateCheck( lastUpdatedMap, rule, context ) ) + { + Double leftSide = context.getExpressionService().getExpressionValue( rule.getLeftSide(), + currentValueMap, context.getConstantMap(), null, incompleteValues ); + + if ( leftSide != null || Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + Double rightSide = getRightSideValue( sourceX.getSource(), periodTypeX, period, rule, + currentValueMap, sourceDataElements, context ); + + if ( rightSide != null || Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + boolean violation = false; + + if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) + { + violation = (leftSide != null && rightSide == null) + || (leftSide == null && rightSide != null); + } + else if ( leftSide != null && rightSide != null ) + { + violation = !expressionIsTrue( leftSide, rule.getOperator(), rightSide ); + } + + if ( violation ) + { + context.getValidationResults().add( new ValidationResult( period, + sourceX.getSource(), rule, getRounded( zeroIfNull( leftSide ), DECIMALS ), + getRounded( zeroIfNull( rightSide ), DECIMALS ) ) ); + } + + log.trace( "-->Evaluated " + rule.getName() + ": " + + (violation ? "violation" : "OK") + " " + leftSide.toString() + " " + + rule.getOperator() + " " + rightSide.toString() + " (" + + context.getValidationResults().size() + " results)" ); + } + } + } + } + } + } + } + } + } + + /** + * Gets the rules that should be evaluated for a given organisation unit and + * period type. + * + * @param sourceX the organisation unit extended information + * @param periodTypeX the period type extended information + * @param context the alert run context + * @param sourceDataElements all data elements collected for this + * organisation unit + * @return + */ + private Set getRulesBySourceAndPeriodType( OrganisationUnitExtended sourceX, + PeriodTypeExtended periodTypeX, ValidationRunContext context, Collection sourceDataElements ) + { + Set periodTypeRules = new HashSet(); + + for ( ValidationRule rule : periodTypeX.getRules() ) + { + if ( (ValidationRule.RULE_TYPE_VALIDATION.equals( rule.getRuleType() )) ) + { + // For validation-type rules, include only rules where the + // organisation collects all the data elements in the rule. + // But if this is some funny kind of rule with no elements + // (like for testing), include it also. + Collection elements = rule.getCurrentDataElements(); + if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) ) + { + periodTypeRules.add( rule ); + } + } + else + { + // For monitoring-type rules, include only rules for this + // organisation's unit level. + // The organisation may not be configured for the data elements + // because they could be aggregated from a lower level. + if ( rule.getOrganisationUnitLevel() == sourceX.getLevel() ) + { + periodTypeRules.add( rule ); + } + } + } + + return periodTypeRules; + } + + + /** + * Checks to see if the evaluation should go further for this + * evaluationRule, after the "current" data to evaluate has been fetched. + * For INTERACTIVE runs, we always go further (always return true.) For + * ALERT runs, we go further only if something has changed since the last + * successful alert run -- either the rule definition or one of the + * "current" data element / option values. + * + * @param lastUpdatedMap when each data value was last updated + * @param rule the rule that may be evaluated + * @param context the evaluation run context + * @return true if the rule should be evaluated with this data, false if not + */ + private boolean evaluateCheck( Map lastUpdatedMap, ValidationRule rule, + ValidationRunContext context ) + { + boolean evaluate = true; // Assume true for now. + + if ( ValidationRunType.ALERT == context.getRunType() ) + { + if ( context.getLastAlertRun() != null ) // True if no previous alert run + { + if ( rule.getLastUpdated().before( context.getLastAlertRun() ) ) + { + // Get the "current" DataElementOperands from this rule: + // Left+Right sides for VALIDATION, Left side only for + // MONITORING + Collection deos = context.getExpressionService().getOperandsInExpression( + rule.getLeftSide().getExpression() ); + if ( ValidationRule.RULE_TYPE_VALIDATION == rule.getRuleType() ) + { + // Make a copy so we can add to it. + deos = new HashSet( deos ); + + deos.addAll( context.getExpressionService().getOperandsInExpression( rule.getRightSide().getExpression() ) ); + } + + // Return true if any data is more recent than the last + // ALERT run, otherwise return false. + evaluate = false; + for ( DataElementOperand deo : deos ) + { + Date lastUpdated = lastUpdatedMap.get( deo ); + if ( lastUpdated != null && lastUpdated.after( context.getLastAlertRun() ) ) + { + evaluate = true; // True if new/updated data. + break; + } + } + } + } + } + return evaluate; + } + + + /** + * Gets the data elements for which values should be fetched recursively if + * they are not collected for an organisation unit. + * + * @param rules ValidationRules to be evaluated + * @return the data elements to fetch recursively + */ + private Set getRecursiveCurrentDataElements( Set rules ) + { + Set recursiveCurrentDataElements = new HashSet(); + + for ( ValidationRule rule : rules ) + { + if ( ValidationRule.RULE_TYPE_MONITORING.equals( rule.getRuleType() ) + && rule.getCurrentDataElements() != null ) + { + recursiveCurrentDataElements.addAll( rule.getCurrentDataElements() ); + } + } + + return recursiveCurrentDataElements; + } + + /** + * Returns the right-side evaluated value of the validation rule. + * + * @param source organisation unit being evaluated + * @param periodTypeX period type being evaluated + * @param period period being evaluated + * @param rule ValidationRule being evaluated + * @param currentValueMap current values already fetched + * @param sourceDataElements the data elements collected by the organisation + * unit + * @param context the validation run context + * @return the right-side value + */ + private Double getRightSideValue( OrganisationUnit source, PeriodTypeExtended periodTypeX, Period period, + ValidationRule rule, Map currentValueMap, + Collection sourceDataElements, ValidationRunContext context ) + { + Double rightSideValue = null; + + // If ruleType is VALIDATION, the right side is evaluated using the same + // (current) data values. If ruleType is MONITORING but there are no + // data elements in the right side, then it doesn't matter what data + // values we use, so just supply the current data values in order to + // evaluate the (constant) expression. + + if ( ValidationRule.RULE_TYPE_VALIDATION.equals( rule.getRuleType() ) + || rule.getRightSide().getDataElementsInExpression().isEmpty() ) + { + rightSideValue = context.getExpressionService().getExpressionValue( rule.getRightSide(), + currentValueMap, context.getConstantMap(), null ); + } + else + // ruleType equals MONITORING, and there are some data elements in the + // right side expression + { + CalendarPeriodType calendarPeriodType = ( CalendarPeriodType ) period.getPeriodType(); + Collection rightSidePeriodTypes = context.getRuleXMap().get( rule ).getAllowedPastPeriodTypes(); + List sampleValues = new ArrayList(); + Calendar yearlyCalendar = PeriodType.createCalendarInstance( period.getStartDate() ); + int annualSampleCount = rule.getAnnualSampleCount() == null ? 0 : rule.getAnnualSampleCount(); + int sequentialSampleCount = rule.getSequentialSampleCount() == null ? 0 : rule + .getSequentialSampleCount(); + + for ( int annualCount = 0; annualCount <= annualSampleCount; annualCount++ ) + { + // Defensive copy because createPeriod mutates Calendar. + Calendar calCopy = PeriodType.createCalendarInstance( yearlyCalendar.getTime() ); + + // To track the period at the same time in preceding years. + Period yearlyPeriod = calendarPeriodType.createPeriod( calCopy ); + + // For past years, fetch the period at the same time of year + // as this period, and any periods after this period within the + // sequentialPeriod limit. For the year of the stating period, + // we will only fetch previous sequential periods. + + if ( annualCount > 0 ) + { + // Fetch the period at the same time of year as the + // starting period. + evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, yearlyPeriod, + rule, sourceDataElements, context ); + + // Fetch the sequential periods after this prior-year + // period. + Period sequentialPeriod = new Period( yearlyPeriod ); + for ( int sequentialCount = 0; sequentialCount < sequentialSampleCount; sequentialCount++ ) + { + sequentialPeriod = calendarPeriodType.getNextPeriod( sequentialPeriod ); + evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, + sequentialPeriod, rule, sourceDataElements, context ); + } + } + + // Fetch the seqential periods before this period (both this + // year and past years): + Period sequentialPeriod = new Period( yearlyPeriod ); + for ( int sequentialCount = 0; sequentialCount < sequentialSampleCount; sequentialCount++ ) + { + sequentialPeriod = calendarPeriodType.getPreviousPeriod( sequentialPeriod ); + evaluateRightSidePeriod( periodTypeX, sampleValues, source, rightSidePeriodTypes, + sequentialPeriod, rule, sourceDataElements, context ); + } + + // Move to the previous year: + yearlyCalendar.set( Calendar.YEAR, yearlyCalendar.get( Calendar.YEAR ) - 1 ); + } + + rightSideValue = rightSideAverage( rule, sampleValues, annualSampleCount, sequentialSampleCount ); + } + return rightSideValue; + } + + /** + * Evaluates the right side of a monitoring-type validation rule for a given + * organisation unit and period, and adds the value to a list of sample + * values. + * + * Note that for a monitoring-type rule, evaluating the right side + * expression can result in sampling multiple periods and/or child + * organisation units. + * + * @param periodTypeX the period type extended information + * @param sampleValues the list of sample values to add to + * @param source the organisation unit + * @param allowedPeriodTypes the period types in which the data may exist + * @param period the main period for the validation rule evaluation + * @param rule the monitoring-type rule being evaluated + * @param sourceDataElements the data elements configured for this + * organisation unit + * @param context the evaluation run context + */ + private void evaluateRightSidePeriod( PeriodTypeExtended periodTypeX, List sampleValues, + OrganisationUnit source, Collection allowedPeriodTypes, Period period, ValidationRule rule, + Collection sourceDataElements, ValidationRunContext context ) + { + Period periodInstance = context.getPeriodService().getPeriod( period.getStartDate(), period.getEndDate(), + period.getPeriodType() ); + + if ( periodInstance != null ) + { + Set dataElements = rule.getRightSide().getDataElementsInExpression(); + Set incompleteValues = new HashSet(); + Map dataValueMap = getDataValueMapRecursive( periodTypeX, dataElements, + sourceDataElements, dataElements, allowedPeriodTypes, period, source, null, incompleteValues ); + Double value = context.getExpressionService().getExpressionValue( rule.getRightSide(), dataValueMap, + context.getConstantMap(), null, incompleteValues ); + + if ( value != null ) + { + sampleValues.add( value ); + } + } + } + + /** + * Finds the average right-side sample value. This is used as the right-side + * expression value to evaluate a monitoring-type rule. + * + * @param rule monitoring-type rule being evaluated + * @param sampleValues sample values actually collected + * @param annualSampleCount number of annual samples tried for + * @param sequentialSampleCount number of sequential samples tried for + * @return + */ + Double rightSideAverage( ValidationRule rule, List sampleValues, int annualSampleCount, + int sequentialSampleCount ) + { + // Find the expected sample count for the last period of its type in the + // database: sequentialSampleCount for the immediately preceding periods + // in this year and for every past year: one sample for the same period + // in that year, plus sequentialSampleCounts before and after. + Double average = null; + if ( !sampleValues.isEmpty() ) + { + int expectedSampleCount = sequentialSampleCount + annualSampleCount * (1 + 2 * sequentialSampleCount); + int highOutliers = rule.getHighOutliers() == null ? 0 : rule.getHighOutliers(); + int lowOutliers = rule.getLowOutliers() == null ? 0 : rule.getLowOutliers(); + + // If we had fewer than the expected number of samples, then scale + // back + if ( highOutliers + lowOutliers > sampleValues.size() ) + { + highOutliers = (highOutliers * sampleValues.size()) / expectedSampleCount; + lowOutliers = (lowOutliers * sampleValues.size()) / expectedSampleCount; + } + + // If we (still) have any high and/or low outliers to remove, then + // sort the sample values and remove the high and/or low outliers. + if ( highOutliers + lowOutliers > 0 ) + { + Collections.sort( sampleValues ); + log.trace( "Removing " + highOutliers + " high and " + lowOutliers + " low outliers from " + + Arrays.toString( sampleValues.toArray() ) ); + sampleValues = sampleValues.subList( lowOutliers, sampleValues.size() - highOutliers ); + log.trace( "Result: " + Arrays.toString( sampleValues.toArray() ) ); + } + Double sum = 0.0; + for ( Double sample : sampleValues ) + { + sum += sample; + } + average = sum / sampleValues.size(); + } + return average; + } + + /** + * Gets data values for a given organisation unit and period, recursing if + * necessary to sum the values from child organisation units. + * + * @param periodTypeX period type which we are evaluating + * @param ruleDataElements data elements configured for the rule + * @param sourceDataElements data elements configured for the organisation + * unit + * @param recursiveDataElements data elements for which we will recurse if + * necessary + * @param allowedPeriodTypes all the periods in which we might find the data + * values + * @param period period in which we are looking for values + * @param source organisation unit for which we are looking for values + * @param lastUpdatedMap map showing when each data values was last updated + * @param incompleteValues ongoing list showing which values were found but + * not from all children + * @return the map of values found + */ + private Map getDataValueMapRecursive( PeriodTypeExtended periodTypeX, + Collection ruleDataElements, Collection sourceDataElements, + Set recursiveDataElements, Collection allowedPeriodTypes, Period period, + OrganisationUnit source, Map lastUpdatedMap, Set incompleteValues ) + { + Set dataElementsToGet = new HashSet( ruleDataElements ); + dataElementsToGet.retainAll( sourceDataElements ); + log.trace( "getDataValueMapRecursive: source:" + source.getName() + " elementsToGet[" + + dataElementsToGet.size() + "] allowedPeriodTypes[" + allowedPeriodTypes.size() + "]" ); + + Map dataValueMap; + + if ( dataElementsToGet.isEmpty() ) + { + // We still might get something recursively + dataValueMap = new HashMap(); + } + else + { + dataValueMap = context.getDataValueService().getDataValueMap( dataElementsToGet, period.getStartDate(), source, + allowedPeriodTypes, lastUpdatedMap ); + } + + // See if there are any data elements we need to get recursively: + Set recursiveDataElementsNeeded = new HashSet( recursiveDataElements ); + recursiveDataElementsNeeded.removeAll( dataElementsToGet ); + if ( !recursiveDataElementsNeeded.isEmpty() ) + { + int childCount = 0; + Map childValueCounts = new HashMap(); + + for ( OrganisationUnit child : source.getChildren() ) + { + Collection childDataElements = periodTypeX.getSourceDataElements().get( child ); + Map childMap = getDataValueMapRecursive( periodTypeX, + recursiveDataElementsNeeded, childDataElements, recursiveDataElementsNeeded, allowedPeriodTypes, + period, child, lastUpdatedMap, incompleteValues ); + + for ( DataElementOperand deo : childMap.keySet() ) + { + Double baseValue = dataValueMap.get( deo ); + dataValueMap.put( deo, baseValue == null ? childMap.get( deo ) : baseValue + childMap.get( deo ) ); + + Integer childValueCount = childValueCounts.get( deo ); + childValueCounts.put( deo, childValueCount == null ? 1 : childValueCount + 1 ); + } + + childCount++; + } + + for ( Map.Entry entry : childValueCounts.entrySet() ) + { + if ( childCount != entry.getValue() ) + { + // Remember that we found this DataElementOperand value + // in some but not all children + incompleteValues.add( entry.getKey() ); + } + } + } + + return dataValueMap; + } + +}