=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRuleService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRuleService.java 2013-10-08 17:20:57 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationRuleService.java 2013-10-14 20:00:00 +0000 @@ -92,11 +92,6 @@ */ Collection validate( Date startDate, Date endDate, OrganisationUnit source ); - /** - * For a nightly alert run, run validation tests and notify users as requested of any validations. - */ - void alertRun(); - // ------------------------------------------------------------------------- // ValidationRule // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java 2013-10-13 16:15:28 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/DefaultValidationRuleService.java 2013-10-14 20:00:00 +0000 @@ -127,13 +127,6 @@ this.periodService = periodService; } - private OrganisationUnitService organisationUnitService; - - public void setOrganisationUnitService( OrganisationUnitService organisationUnitService ) - { - this.organisationUnitService = organisationUnitService; - } - private DataValueService dataValueService; public void setDataValueService( DataValueService dataValueService ) @@ -148,20 +141,6 @@ this.constantService = constantService; } - private SystemSettingManager systemSettingManager; - - public void setSystemSettingManager( SystemSettingManager systemSettingManager ) - { - this.systemSettingManager = systemSettingManager; - } - - private MessageService messageService; - - public void setMessageService( MessageService messageService ) - { - this.messageService = messageService; - } - private I18nService i18nService; public void setI18nService( I18nService service ) @@ -178,7 +157,8 @@ log.info( "Validate startDate=" + startDate + " endDate=" + endDate + " sources[" + sources.size() + "]" ); Collection periods = periodService.getPeriodsBetweenDates( startDate, endDate ); Collection rules = getAllValidationRules(); - return validateInternal( sources, periods, rules, ValidationRunType.INTERACTIVE, null ); + return Validator.validate( sources, periods, rules, ValidationRunType.INTERACTIVE, null, + constantService, expressionService, periodService, dataValueService ); } public Collection validate( Date startDate, Date endDate, Collection sources, @@ -187,7 +167,8 @@ log.info( "Validate startDate=" + startDate + " endDate=" + endDate + " sources[" + sources.size() + "] group=" + group.getName() ); Collection periods = periodService.getPeriodsBetweenDates( startDate, endDate ); Collection rules = group.getMembers(); - return validateInternal( sources, periods, rules, ValidationRunType.INTERACTIVE, null ); + return Validator.validate( sources, periods, rules, ValidationRunType.INTERACTIVE, null, + constantService, expressionService, periodService, dataValueService ); } public Collection validate( Date startDate, Date endDate, OrganisationUnit source ) @@ -195,7 +176,10 @@ log.info( "Validate startDate=" + startDate + " endDate=" + endDate + " source=" + source.getName() ); Collection periods = periodService.getPeriodsBetweenDates( startDate, endDate ); Collection rules = getAllValidationRules(); - return validateInternal( source, periods, rules, ValidationRunType.INTERACTIVE, null ); + Collection sources = new HashSet(); + sources.add( source ); + return Validator.validate( sources, periods, rules, ValidationRunType.INTERACTIVE, null, + constantService, expressionService, periodService, dataValueService ); } public Collection validate( DataSet dataSet, Period period, OrganisationUnit source ) @@ -215,45 +199,10 @@ rules = getValidationTypeRulesForDataElements( dataSet.getDataElements() ); } - return validateInternal( source, periods, rules, ValidationRunType.INTERACTIVE, null ); - } - - // TODO: Schedule this to run every night. - public void alertRun() - { - // Find all the rules belonging to groups that will send alerts to user roles. - Set rules = new HashSet(); - for ( ValidationRuleGroup validationRuleGroup : getAllValidationRuleGroups() ) - { - Collection userRolesToAlert = validationRuleGroup.getUserAuthorityGroupsToAlert(); - if ( userRolesToAlert != null && !userRolesToAlert.isEmpty() ) - { - rules.addAll( validationRuleGroup.getMembers() ); - } - } - - Collection sources = organisationUnitService.getAllOrganisationUnits(); - Collection periods = getAlertPeriodsFromRules( rules ); - Date lastAlertRun = (Date) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_LAST_ALERT_RUN ); - - // Any database changes after this moment will contribute to the next run. - - Date thisAlertRun = new Date(); - - log.info( "alertRun sources[" + sources.size() + "] periods[" + periods.size() + "] rules[" + rules.size() - + "] last run " + lastAlertRun == null ? "(none)" : lastAlertRun ); - - Collection results = validateInternal( sources, periods, rules, ValidationRunType.ALERT, - lastAlertRun ); - - log.trace( "alertRun() results[" + results.size() + "]" ); - - if ( !results.isEmpty() ) - { - postAlerts( results, thisAlertRun ); // Alert the users. - } - - systemSettingManager.saveSystemSetting( SystemSettingManager.KEY_LAST_ALERT_RUN, thisAlertRun ); + Collection sources = new HashSet(); + sources.add( source ); + return Validator.validate( sources, periods, rules, ValidationRunType.INTERACTIVE, null, + constantService, expressionService, periodService, dataValueService ); } // ------------------------------------------------------------------------- @@ -261,117 +210,6 @@ // ------------------------------------------------------------------------- /** - * Evaluates validation rules for a single organisation unit. - * - * @param source the organisation unit in which to run the validation rules - * @param periods the periods of data to check - * @param rules the ValidationRules to evaluate - * @param runType whether this is an interactive or alert run - * @param lastAlertRun date/time of the most recent successful alerts run - * (needed only for alert run) - * @return a collection of any validations that were found - */ - private Collection validateInternal( OrganisationUnit source, Collection periods, - Collection rules, ValidationRunType runType, Date lastAlertRun ) - { - Collection sources = new HashSet(); - sources.add( source ); - return validateInternal( sources, periods, rules, ValidationRunType.INTERACTIVE, null ); - } - - /** - * Evaluates validation rules for a collection of organisation units. - * This method breaks the job down by organisation unit. It assigns the - * evaluation for each organisation unit to a task that can be evaluated - * independently in a multithreaded environment. - * - * @param sources the organisation units in which to run the validation - * rules - * @param periods the periods of data to check - * @param rules the ValidationRules to evaluate - * @param runType whether this is an INTERACTIVE or ALERT run - * @param lastAlertRun date/time of the most recent successful alerts run - * (needed only for alert run) - * @return a collection of any validations that were found - */ - private Collection validateInternal( Collection sources, - Collection periods, Collection rules, ValidationRunType runType, Date lastAlertRun ) - { - ValidationRunContext context = ValidationRunContext.getNewValidationRunContext( sources, periods, rules, - constantService.getConstantMap(), ValidationRunType.ALERT, lastAlertRun, - expressionService, periodService, dataValueService); - - int threadPoolSize = SystemUtils.getCpuCores(); - - if ( threadPoolSize > 2 ) - { - threadPoolSize--; - } - if ( threadPoolSize > sources.size() ) - { - threadPoolSize = sources.size(); - } - - ExecutorService executor = Executors.newFixedThreadPool( threadPoolSize ); - - for ( OrganisationUnitExtended sourceX : context.getSourceXs() ) - { - Runnable worker = new ValidationWorkerThread( sourceX, context ); - executor.execute( worker ); - } - - executor.shutdown(); - - try - { - executor.awaitTermination( 23, TimeUnit.HOURS ); - } - catch ( InterruptedException e ) - { - executor.shutdownNow(); - } - return context.getValidationResults(); - } - - /** - * For an alert run, gets the current and most periods to search, based on - * the period types from the rules to run. - * - * @param rules the ValidationRules to be evaluated on this alert run - * @return periods to search for new alerts - */ - private Collection getAlertPeriodsFromRules( Collection rules ) - { - Set periods = new HashSet(); - - // Construct a set of all period types found in validation rules. - Set rulePeriodTypes = new HashSet(); - for ( ValidationRule rule : rules ) - { - // (Note that we have to get periodType from periodService, - // otherwise the ID will not be present.) - rulePeriodTypes.add( periodService.getPeriodTypeByName( rule.getPeriodType().getName() ) ); - } - - // For each period type, find the period containing the current date (if - // any), and the most recent previous period. Add whichever one(s) of these - // are present in the database. - for ( PeriodType periodType : rulePeriodTypes ) - { - CalendarPeriodType calendarPeriodType = ( CalendarPeriodType ) periodType; - Period currentPeriod = calendarPeriodType.createPeriod(); - Period previousPeriod = calendarPeriodType.getPreviousPeriod( currentPeriod ); - periods.addAll( periodService.getIntersectingPeriodsByPeriodType( periodType, - previousPeriod.getStartDate(), currentPeriod.getEndDate() ) ); - // Note: If the last successful daily run was more than one day - // ago, we might consider adding some additional periods of type - // DailyPeriodType so we don't miss any alerts. - } - - return periods; - } - - /** * Returns all validation-type rules which have specified data elements * assigned to them. * @@ -437,176 +275,6 @@ return rulesForDataSet; } - - /** - * At the end of an ALERT run, post messages to the users who want to see - * the results. - * - * @param validationResults the set of validation error results - * @param alertRunStart the date/time when the alert run started - */ - private void postAlerts( Collection validationResults, Date alertRunStart ) - { - SortedSet results = new TreeSet( validationResults ); - - // Find out which users receive alerts, and which ValidationRules they - // receive alerts for. - Map> userRulesMap = new HashMap>(); - - for ( ValidationRuleGroup validationRuleGroup : getAllValidationRuleGroups() ) - { - Collection userRolesToAlert = validationRuleGroup.getUserAuthorityGroupsToAlert(); - if ( userRolesToAlert != null && !userRolesToAlert.isEmpty() ) - { - for ( UserAuthorityGroup role : userRolesToAlert ) - { - for ( UserCredentials userCredentials : role.getMembers() ) - { - User user = userCredentials.getUser(); - Collection userRules = userRulesMap.get( user ); - if ( userRules == null ) - { - userRules = new ArrayList(); - userRulesMap.put( user, userRules ); - } - userRules.addAll( validationRuleGroup.getMembers() ); - } - } - } - } - - // We will create one message for each set of users who receive the same - // subset of results. (Not necessarily the same as the set of users who - // receive alerts from the same subset of validation rules -- because - // some of these rules may return no results.) This saves on message - // storage space. - - // TODO: Encapsulate this in another level of Map by the user's - // language, and generate a message for each combination of ( target - // language, set of results ) - Map, Set> messageMap = new HashMap, Set>(); - - for ( User user : userRulesMap.keySet() ) - { - // For each user who receives alerts, find the subset of results - // from this run. - Collection userRules = userRulesMap.get( user ); - List userResults = new ArrayList(); - Map importanceSummary = new HashMap(); - - for ( ValidationResult result : results ) - { - if ( userRules.contains( result.getValidationRule() ) ) - { - userResults.add( result ); - String importance = result.getValidationRule().getImportance(); - Integer importanceCount = importanceSummary.get( importance ); - if ( importanceCount == null ) - { - importanceSummary.put( importance, 1 ); - } - else - { - importanceSummary.put( importance, importanceCount + 1 ); - } - } - } - - // Group this user with other users who have the same subset of - // results. - if ( !userResults.isEmpty() ) - { - Set messageReceivers = messageMap.get( userResults ); - if ( messageReceivers == null ) - { - messageReceivers = new HashSet(); - messageMap.put( userResults, messageReceivers ); - } - messageReceivers.add( user ); - } - } - - // For each unique subset of results, send a message to all users - // receiving that subset of results. - for ( Map.Entry, Set> entry : messageMap.entrySet() ) - { - sendAlertmessage( entry.getKey(), entry.getValue(), alertRunStart ); - } - } - - /** - * Generate and send an alert message containing alert results to a set of - * users. - * - * @param results results to put in this message - * @param users users to receive these results - * @param alertRunStart date/time when the alert run started - */ - private void sendAlertmessage( List results, Set users, Date alertRunStart ) - { - StringBuilder messageBuilder = new StringBuilder(); - SimpleDateFormat dateTimeFormatter = new SimpleDateFormat( "yyyy-MM-dd HH:mm" ); - - // Count the number of messages of each importance type. - Map importanceCountMap = new HashMap(); - for ( ValidationResult result : results ) - { - Integer importanceCount = importanceCountMap.get( result.getValidationRule().getImportance() ); - importanceCountMap.put( result.getValidationRule().getImportance(), importanceCount == null ? 1 - : importanceCount + 1 ); - } - - // Construct the subject line. - String subject = "DHIS alerts as of " + dateTimeFormatter.format( alertRunStart ) + " - priority High " - + (importanceCountMap.get( "high" ) == null ? 0 : importanceCountMap.get( "high" )) + ", Medium " - + (importanceCountMap.get( "medium" ) == null ? 0 : importanceCountMap.get( "medium" )) + ", Low " - + (importanceCountMap.get( "low" ) == null ? 0 : importanceCountMap.get( "low" )); - - // Construct the text of the message. - messageBuilder - .append( "" ) - .append( "" ).append( "" ) - .append( "" ).append( subject ).append( "
" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ) - .append( "" ); - - for ( ValidationResult result : results ) - { - ValidationRule rule = result.getValidationRule(); - - messageBuilder - .append( "" ) - .append( "" ); - } - - messageBuilder - .append( "
Organisation UnitPeriodImportanceLeft side descriptionValueOperatorValueRight side description
" ).append( result.getSource().getName() ).append( "<\td>" ) - .append( "" ).append( result.getPeriod().getName() ).append( "<\td>" ) - .append( "" ).append( rule.getImportance() ).append( "<\td>" ) - .append( "" ).append( rule.getLeftSide().getDescription() ).append( "<\td>" ) - .append( "" ).append( result.getLeftsideValue() ).append( "<\td>" ) - .append( "" ).append( rule.getOperator().toString() ).append( "<\td>" ) - .append( "" ).append( result.getRightsideValue() ).append( "<\td>" ) - .append( "" ).append( rule.getRightSide().getDescription() ).append( "<\td>" ) - .append( "
" ) - .append( "" ) - .append( "" ); - - String messageText = messageBuilder.toString(); - - log.info( "postUserResults() users[" + users.size() + "] subject " + subject ); - messageService.sendMessage( subject, messageText, null, users ); - } - // ------------------------------------------------------------------------- // ValidationRule CRUD operations // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java 2013-10-09 05:16:53 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/OrganisationUnitExtended.java 2013-10-14 20:00:00 +0000 @@ -48,14 +48,17 @@ public class OrganisationUnitExtended { private OrganisationUnit source; + + private boolean toBeValidated; private Collection children; private int level; - public OrganisationUnitExtended( OrganisationUnit source ) + public OrganisationUnitExtended( OrganisationUnit source, boolean toBeValidated ) { this.source = source; + this.toBeValidated = toBeValidated; children = new HashSet( source.getChildren() ); level = source.getOrganisationUnitLevel(); } @@ -63,8 +66,9 @@ public String toString() { return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) - .append( "\n name", source.getName() ).append( "\n children[", children.size() + "]" ) - .append( "\n level", level ).toString(); + .append( "\n name", source.getName() ) + .append( "\n children[", children.size() + "]" ) + .append( "\n level", level ).toString(); } // ------------------------------------------------------------------------- @@ -75,6 +79,10 @@ return source; } + public boolean getToBeValidated() { + return toBeValidated; + } + public Collection getChildren() { return children; } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java 2013-10-09 05:16:53 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/PeriodTypeExtended.java 2013-10-14 20:00:00 +0000 @@ -81,12 +81,12 @@ public String toString() { return new ToStringBuilder( this, ToStringStyle.SHORT_PREFIX_STYLE ) - .append( "\n periodType", periodType ) - .append( "\n periods", (Arrays.toString( periods.toArray() )) ) - .append( "\n rules", (Arrays.toString( rules.toArray() )) ) - .append( "\n dataElements", (Arrays.toString( dataElements.toArray() )) ) - .append( "\n allowedPeriodTypes", (Arrays.toString( allowedPeriodTypes.toArray() )) ) - .append( "\n sourceDataElements", "[" + sourceDataElements.size() + "]" ).toString(); + .append( "\n periodType", periodType ) + .append( "\n periods", (Arrays.toString( periods.toArray() )) ) + .append( "\n rules", (Arrays.toString( rules.toArray() )) ) + .append( "\n dataElements", (Arrays.toString( dataElements.toArray() )) ) + .append( "\n allowedPeriodTypes", (Arrays.toString( allowedPeriodTypes.toArray() )) ) + .append( "\n sourceDataElements", "[" + sourceDataElements.size() + "]" ).toString(); } // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java 2013-10-13 09:59:35 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java 2013-10-14 20:00:00 +0000 @@ -81,6 +81,8 @@ private Map ruleXMap; private Collection sourceXs; + + private int countOfSourcesToValidate; private Collection validationResults; @@ -101,8 +103,8 @@ .append( "\n runType", runType ).append( "\n lastAlertRun", lastAlertRun ) .append( "\n constantMap", "[" + constantMap.size() + "]" ) .append( "\n ruleXMap", "[" + ruleXMap.size() + "]" ) - .append( "\n sourceXs", Arrays.toString( sourceXs.toArray() ) ) - .append( "\n validationResults", Arrays.toString( validationResults.toArray() ) ).toString(); + .append( "\n sourceXs", Arrays.toString( sourceXs.toArray() ) ) + .append( "\n validationResults", Arrays.toString( validationResults.toArray() ) ).toString(); } /** @@ -138,22 +140,54 @@ /** * Initializes context values based on sources, periods and rules * - * @param sources - * @param periods - * @param rules + * @param sources organisation units to evaluate for rules + * @param periods periods for validation + * @param rules validation rules for validation */ private void initialize( Collection sources, Collection periods, Collection rules ) { - boolean monitoringRulesPresent = false; - - // Group the periods by period type. - for ( Period period : periods ) + addPeriodsToContext( periods ); + + boolean monitoringRulesPresent = addRulesToContext ( rules ); + + removeAnyUnneededPeriodTypes(); + + addSourcesToContext( sources, true ); + + countOfSourcesToValidate = sources.size(); + + if ( monitoringRulesPresent ) { - PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( period.getPeriodType() ); - periodTypeX.getPeriods().add( period ); + Set otherDescendants = getAllOtherDescendants( sources ); + addSourcesToContext( otherDescendants, false ); } - + } + + /** + * Adds Periods to the context, grouped by period type. + * + * @param periods Periods to group and add + */ + private void addPeriodsToContext ( Collection periods ) + { + for ( Period period : periods ) + { + PeriodTypeExtended periodTypeX = getOrCreatePeriodTypeExtended( period.getPeriodType() ); + periodTypeX.getPeriods().add( period ); + } + } + + /** + * Adds validation rules to the context. + * + * @param rules validation rules to add + * @return true if there were some monitoring-type rules, false otherwise. + */ + private boolean addRulesToContext ( Collection rules ) + { + boolean monitoringRulesPresent = false; + for ( ValidationRule rule : rules ) { if ( ValidationRule.RULE_TYPE_MONITORING.equals( rule.getRuleType() ) ) @@ -187,34 +221,91 @@ ValidationRuleExtended ruleX = new ValidationRuleExtended( rule, allowedPastPeriodTypes ); ruleXMap.put( rule, ruleX ); } + return monitoringRulesPresent; + } - // We only need to keep period types that are selected and also used by - // rules that are selected. + /** + * Removes any period types that don't have rules assigned to them. + */ + private void removeAnyUnneededPeriodTypes() + { // Start by making a defensive copy so we can delete while iterating. Set periodTypeXs = new HashSet( periodTypeExtendedMap.values() ); for ( PeriodTypeExtended periodTypeX : periodTypeXs ) { - if ( periodTypeX.getPeriods().isEmpty() || periodTypeX.getRules().isEmpty() ) + if ( periodTypeX.getRules().isEmpty() ) { periodTypeExtendedMap.remove( periodTypeX.getPeriodType() ); } } - - // If we have some monitoring rules, then make sure that we add all - // the descendants of each source to our collection of sources. - // This is so we can recurse if needed to find monitoring rule values. - if ( monitoringRulesPresent ) - { - for ( OrganisationUnit source : new HashSet( sources ) ) + } + + /** + * Finds all organisation unit descendants that are not in a given + * collection of organisation units. This is needed for monitoring-type + * rules, because the data values for the rules may need to be aggregated + * from the organisation unit's descendants. + * + * The descendants will likely be there anyway for a run including + * monitoring-type rules, because an interactive run containing + * monitoring-type rules should select an entire subtree, and a + * scheduled alert run will contain all organisation units. But check + * just to be sure, and find any that may be missing. This makes sure + * that some of the tests will work, and may be required for some + * future features to work. + * + * @param sources organisation units whose descendants to check + * @return all other descendants who need to be added who were not + * in the original list + */ + private Set getAllOtherDescendants( Collection sources ) + { + Set allOtherDescendants = new HashSet(); + for ( OrganisationUnit source : sources ) + { + getOtherDescendantsRecursive( source, sources, allOtherDescendants ); + } + return allOtherDescendants; + } + + /** + * If the children of this organisation unit are not in the collection, then + * add them and all their descendants if needed. + * + * @param source organisation unit whose children to check + * @param sources organisation units in the initial list + * @param allOtherDescendants list of organisation unit descendants we + * need to add + */ + private void getOtherDescendantsRecursive( OrganisationUnit source, Collection sources, + Set allOtherDescendants ) + { + for ( OrganisationUnit child : source.getChildren() ) + { + if ( !sources.contains( child ) && !allOtherDescendants.contains( child ) ) { - addSourceRecursive( sources, source ); + allOtherDescendants.add( child ); + getOtherDescendantsRecursive( child, sources, allOtherDescendants ); } } + } + /** + * Adds a collection of organisation units to the validation run context. + * + * @param sources organisation units to add + * @param ruleCheckThisSource true if these organisation units should be + * evaluated with validation rules, false if not. (This is false when + * adding descendants of organisation units for the purpose of getting + * aggregated expression values from descendants, but these organisation + * units are not in the main list to be evaluated.) + */ + private void addSourcesToContext ( Collection sources, boolean ruleCheckThisSource ) + { // Get the information we need for each source. for ( OrganisationUnit source : sources ) { - OrganisationUnitExtended sourceX = new OrganisationUnitExtended( source ); + OrganisationUnitExtended sourceX = new OrganisationUnitExtended( source, ruleCheckThisSource ); sourceXs.add( sourceX ); Map> sourceElementsMap = source.getDataElementsInDataSetsByPeriodType(); @@ -232,26 +323,7 @@ } } } - - /** - * If the children of this organisation unit are not in the collection, then - * add them and all their descendants if needed. - * - * @param sources organisation units to test and add to - * @param source organisation unit whose children to check - */ - private void addSourceRecursive( Collection sources, OrganisationUnit source ) - { - for ( OrganisationUnit child : source.getChildren() ) - { - if ( !sources.contains( child ) ) - { - sources.add( child ); - addSourceRecursive( sources, child ); - } - } - } - + /** * Gets the PeriodTypeExtended from the context object. If not found, * creates a new PeriodTypeExtended object, puts it into the context object, @@ -334,6 +406,11 @@ return sourceXs; } + public int getCountOfSourcesToValidate() + { + return countOfSourcesToValidate; + } + public Collection getValidationResults() { return validationResults; === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/Validator.java 2013-10-14 20:00:00 +0000 @@ -0,0 +1,127 @@ +package org.hisp.dhis.validation; + +import java.util.Collection; +import java.util.Date; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.hisp.dhis.constant.ConstantService; +import org.hisp.dhis.datavalue.DataValueService; +import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.system.util.SystemUtils; + +/* + * Copyright (c) 2004-2013, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Evaluates validation rules. + * + * @author Jim Grace + */ +public class Validator +{ + + /** + * Evaluates validation rules for a collection of organisation units. + * This method breaks the job down by organisation unit. It assigns the + * evaluation for each organisation unit to a task that can be evaluated + * independently in a multithreaded environment. + * + * @param sources the organisation units in which to run the validation + * rules + * @param periods the periods of data to check + * @param rules the ValidationRules to evaluate + * @param runType whether this is an INTERACTIVE or ALERT run + * @param lastAlertRun date/time of the most recent successful alerts run + * (needed only for alert run) + * @param constantService Constant Service reference + * @param expressionService Expression Service reference + * @param periodService Period Service reference + * @param dataValueService Data Value Service reference + * @return a collection of any validations that were found + */ + public static Collection validate( Collection sources, + Collection periods, Collection rules, ValidationRunType runType, Date lastAlertRun, + ConstantService constantService, ExpressionService expressionService, PeriodService periodService, DataValueService dataValueService) + { + ValidationRunContext context = ValidationRunContext.getNewValidationRunContext( sources, periods, rules, + constantService.getConstantMap(), ValidationRunType.ALERT, lastAlertRun, + expressionService, periodService, dataValueService); + + int threadPoolSize = getThreadPoolSize( context ); + ExecutorService executor = Executors.newFixedThreadPool( threadPoolSize ); + + for ( OrganisationUnitExtended sourceX : context.getSourceXs() ) + { + if ( sourceX.getToBeValidated() ) + { + Runnable worker = new ValidatorThread( sourceX, context ); + executor.execute( worker ); + } + } + + executor.shutdown(); + + try + { + executor.awaitTermination( 23, TimeUnit.HOURS ); + } + catch ( InterruptedException e ) + { + executor.shutdownNow(); + } + return context.getValidationResults(); + } + + /** + * Determines how many threads we should use for testing validation rules. + * + * @param context validation run context + * @return number of threads we should use for testing validation rules + */ + private static int getThreadPoolSize( ValidationRunContext context ) + { + int threadPoolSize = SystemUtils.getCpuCores(); + + if ( threadPoolSize > 2 ) + { + threadPoolSize--; + } + if ( threadPoolSize > context.getCountOfSourcesToValidate() ) + { + threadPoolSize = context.getCountOfSourcesToValidate(); + } + + return threadPoolSize; + } + +} === renamed file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationWorkerThread.java' => 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidationWorkerThread.java 2013-10-11 12:58:30 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2013-10-14 20:00:00 +0000 @@ -57,20 +57,20 @@ /** * Runs a validation task on a thread within a multi-threaded validation run. * - * Each thread looks for validation results in a different organisation unit. + * Each task looks for validation results in a different organisation unit. * * @author Jim Grace */ -public class ValidationWorkerThread +public class ValidatorThread implements Runnable { - private static final Log log = LogFactory.getLog( ValidationWorkerThread.class ); + private static final Log log = LogFactory.getLog( ValidatorThread.class ); private OrganisationUnitExtended sourceX; private ValidationRunContext context; - public ValidationWorkerThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) + public ValidatorThread( OrganisationUnitExtended sourceX, ValidationRunContext context ) { this.sourceX = sourceX; this.context = context; === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/scheduling/MonitoringTask.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/scheduling/MonitoringTask.java 2013-10-13 20:16:32 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/scheduling/MonitoringTask.java 2013-10-14 20:00:00 +0000 @@ -31,18 +31,53 @@ import static org.hisp.dhis.system.notification.NotificationLevel.ERROR; import static org.hisp.dhis.system.notification.NotificationLevel.INFO; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.constant.ConstantService; +import org.hisp.dhis.datavalue.DataValueService; +import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.message.MessageService; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.CalendarPeriodType; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.scheduling.TaskId; +import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.notification.Notifier; +import org.hisp.dhis.user.User; +import org.hisp.dhis.user.UserAuthorityGroup; +import org.hisp.dhis.user.UserCredentials; +import org.hisp.dhis.validation.ValidationResult; +import org.hisp.dhis.validation.ValidationRule; +import org.hisp.dhis.validation.ValidationRuleGroup; import org.hisp.dhis.validation.ValidationRuleService; +import org.hisp.dhis.validation.ValidationRunType; +import org.hisp.dhis.validation.Validator; import org.springframework.beans.factory.annotation.Autowired; /** * @author Lars Helge Overland + * @author Jim Grace */ public class MonitoringTask implements Runnable { + private static final Log log = LogFactory.getLog( MonitoringTask.class ); + @Autowired private ValidationRuleService validationRuleService; @@ -50,6 +85,24 @@ private Notifier notifier; @Autowired + private ExpressionService expressionService; + + @Autowired + private PeriodService periodService; + + @Autowired + private OrganisationUnitService organisationUnitService; + + @Autowired + private DataValueService dataValueService; + + @Autowired + private ConstantService constantService; + + @Autowired + private SystemSettingManager systemSettingManager; + + @Autowired private MessageService messageService; private TaskId taskId; @@ -70,7 +123,7 @@ try { - validationRuleService.alertRun(); + alertRun(); notifier.notify( taskId, INFO, "Monitoring process done", true ); } @@ -83,4 +136,312 @@ throw ex; } } + + // ------------------------------------------------------------------------- + // Support methods + // ------------------------------------------------------------------------- + + /** + * Evaluates all the validation rules that could generate alerts, + * and sends results (if any) to users who should be notified. + */ + private void alertRun() + { + // Find all the rules belonging to groups that will send alerts to user roles. + Set rules = getAlertRules(); + + Collection sources = organisationUnitService.getAllOrganisationUnits(); + Set periods = getAlertPeriodsFromRules( rules ); + Date lastAlertRun = (Date) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_LAST_ALERT_RUN ); + + // Any database changes after this moment will contribute to the next run. + + Date thisAlertRun = new Date(); + + log.info( "alertRun sources[" + sources.size() + "] periods[" + periods.size() + "] rules[" + rules.size() + + "] last run " + lastAlertRun == null ? "(none)" : lastAlertRun ); + + Collection results = Validator.validate( sources, periods, rules, ValidationRunType.ALERT, + lastAlertRun, constantService, expressionService, periodService, dataValueService ); + + log.trace( "alertRun() results[" + results.size() + "]" ); + + if ( !results.isEmpty() ) + { + postAlerts( results, thisAlertRun ); // Alert the users. + } + + systemSettingManager.saveSystemSetting( SystemSettingManager.KEY_LAST_ALERT_RUN, thisAlertRun ); + } + + /** + * Gets all the validation rules that could generate alerts. + * + * @return rules that will generate alerts + */ + private Set getAlertRules() + { + Set rules = new HashSet(); + for ( ValidationRuleGroup validationRuleGroup : validationRuleService.getAllValidationRuleGroups() ) + { + Set userRolesToAlert = validationRuleGroup.getUserAuthorityGroupsToAlert(); + if ( userRolesToAlert != null && !userRolesToAlert.isEmpty() ) + { + rules.addAll( validationRuleGroup.getMembers() ); + } + } + + return rules; + } + + /** + * Gets the current and most recent periods to search, based on + * the period types from the rules to run. + * + * For each period type, return the period containing the current date + * (if any), and the most recent previous period. Add whichever of + * these periods actually exist in the database. + * + * @param rules the ValidationRules to be evaluated on this alert run + * @return periods to search for new alerts + */ + private Set getAlertPeriodsFromRules( Set rules ) + { + Set periods = new HashSet(); + + Set rulePeriodTypes = getPeriodTypesFromRules( rules ); + + for ( PeriodType periodType : rulePeriodTypes ) + { + CalendarPeriodType calendarPeriodType = ( CalendarPeriodType ) periodType; + Period currentPeriod = calendarPeriodType.createPeriod(); + Period previousPeriod = calendarPeriodType.getPreviousPeriod( currentPeriod ); + periods.addAll( periodService.getIntersectingPeriodsByPeriodType( periodType, + previousPeriod.getStartDate(), currentPeriod.getEndDate() ) ); + // Note: If the last successful daily run was more than one day + // ago, we might consider adding some additional periods of type + // DailyPeriodType so we don't miss any alerts. + } + + return periods; + } + + /** + * Gets the Set of period types found in a set of rules. + * + * Note that that we have to get periodType from periodService, + * otherwise the ID will not be present.) + * + * @param rules validation rules of interest + * @return period types contained in those rules + */ + private Set getPeriodTypesFromRules ( Collection rules ) + { + Set rulePeriodTypes = new HashSet(); + + for ( ValidationRule rule : rules ) + { + rulePeriodTypes.add( periodService.getPeriodTypeByName( rule.getPeriodType().getName() ) ); + } + + return rulePeriodTypes; + } + + /** + * At the end of an ALERT run, post messages to the users who want to see + * the results. + * + * Create one message for each set of users who receive the same + * subset of results. (Not necessarily the same as the set of users who + * receive alerts from the same subset of validation rules -- because + * some of these rules may return no results.) This saves on message + * storage space. + * + * The message results are sorted into their natural order. + * + * TODO: Internationalize the messages according to the user's + * preferred language, and generate a message for each combination of + * ( target language, set of results ). + * + * @param validationResults the set of validation error results + * @param alertRunStart the date/time when the alert run started + */ + private void postAlerts( Collection validationResults, Date alertRunStart ) + { + SortedSet results = new TreeSet( validationResults ); + + Map, Set> messageMap = getMessageMap( results ); + + for ( Map.Entry, Set> entry : messageMap.entrySet() ) + { + sendAlertmessage( entry.getKey(), entry.getValue(), alertRunStart ); + } + } + + /** + * Returns a map where the key is a (sorted) list of validation results + * to assemble into a message, and the value is the set of users who + * should receive this message. + * + * @param results all the validation run results + * @return map of result sets to users + */ + private Map, Set> getMessageMap( Set results ) + { + Map> userRulesMap = getUserRulesMap(); + + Map, Set> messageMap = new HashMap, Set>(); + + for ( User user : userRulesMap.keySet() ) + { + // For each user who receives alerts, find the subset of results + // from this run. + Collection userRules = userRulesMap.get( user ); + List userResults = new ArrayList(); + + for ( ValidationResult result : results ) + { + if ( userRules.contains( result.getValidationRule() ) ) + { + userResults.add( result ); + } + } + + // Group this user with any other users who have the same subset + // of results. + if ( !userResults.isEmpty() ) + { + Set messageReceivers = messageMap.get( userResults ); + if ( messageReceivers == null ) + { + messageReceivers = new HashSet(); + messageMap.put( userResults, messageReceivers ); + } + messageReceivers.add( user ); + } + } + return messageMap; + } + + /** + * Constructs a Map where the key is each user who is configured to + * receive alerts, and the value is a list of rules they should receive + * results for. + * + * @return Map from users to sets of rules + */ + private Map> getUserRulesMap() + { + Map> userRulesMap = new HashMap>(); + + for ( ValidationRuleGroup validationRuleGroup : validationRuleService.getAllValidationRuleGroups() ) + { + Collection userRolesToAlert = validationRuleGroup.getUserAuthorityGroupsToAlert(); + if ( userRolesToAlert != null && !userRolesToAlert.isEmpty() ) + { + for ( UserAuthorityGroup role : userRolesToAlert ) + { + for ( UserCredentials userCredentials : role.getMembers() ) + { + User user = userCredentials.getUser(); + Set userRules = userRulesMap.get( user ); + if ( userRules == null ) + { + userRules = new HashSet(); + userRulesMap.put( user, userRules ); + } + userRules.addAll( validationRuleGroup.getMembers() ); + } + } + } + } + return userRulesMap; + } + + /** + * Generate and send an alert message containing a list of validation + * results to a set of users. + * + * @param results results to put in this message + * @param users users to receive these results + * @param alertRunStart date/time when the alert run started + */ + private void sendAlertmessage( List results, Set users, Date alertRunStart ) + { + StringBuilder messageBuilder = new StringBuilder(); + + SimpleDateFormat dateTimeFormatter = new SimpleDateFormat( "yyyy-MM-dd HH:mm" ); + + Map importanceCountMap = countTheResultsByImportanceType( results ); + + String subject = "DHIS alerts as of " + dateTimeFormatter.format( alertRunStart ) + " - priority High " + + (importanceCountMap.get( "high" ) == null ? 0 : importanceCountMap.get( "high" )) + ", Medium " + + (importanceCountMap.get( "medium" ) == null ? 0 : importanceCountMap.get( "medium" )) + ", Low " + + (importanceCountMap.get( "low" ) == null ? 0 : importanceCountMap.get( "low" )); + + messageBuilder + .append( "" ) + .append( "" ).append( "" ) + .append( "" ).append( subject ).append( "
" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ) + .append( "" ); + + for ( ValidationResult result : results ) + { + ValidationRule rule = result.getValidationRule(); + + messageBuilder + .append( "" ) + .append( "" ); + } + + messageBuilder + .append( "
Organisation UnitPeriodImportanceLeft side descriptionValueOperatorValueRight side description
" ).append( result.getSource().getName() ).append( "<\td>" ) + .append( "" ).append( result.getPeriod().getName() ).append( "<\td>" ) + .append( "" ).append( rule.getImportance() ).append( "<\td>" ) + .append( "" ).append( rule.getLeftSide().getDescription() ).append( "<\td>" ) + .append( "" ).append( result.getLeftsideValue() ).append( "<\td>" ) + .append( "" ).append( rule.getOperator().toString() ).append( "<\td>" ) + .append( "" ).append( result.getRightsideValue() ).append( "<\td>" ) + .append( "" ).append( rule.getRightSide().getDescription() ).append( "<\td>" ) + .append( "
" ) + .append( "" ) + .append( "" ); + + String messageText = messageBuilder.toString(); + + log.info( "Alerting users[" + users.size() + "] subject " + subject ); + messageService.sendMessage( subject, messageText, null, users ); + } + + /** + * Counts the results of each importance type, for all the importance + * types that are found within the results. + * + * @param results results to analyze + * @return Map showing the result count for each importance type + */ + private Map countTheResultsByImportanceType ( List results ) + { + Map importanceCountMap = new HashMap(); + + for ( ValidationResult result : results ) + { + Integer importanceCount = importanceCountMap.get( result.getValidationRule().getImportance() ); + + importanceCountMap.put( result.getValidationRule().getImportance(), importanceCount == null ? 1 + : importanceCount + 1 ); + } + + return importanceCountMap; + } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2013-10-13 20:16:32 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2013-10-14 20:00:00 +0000 @@ -433,11 +433,8 @@ - - -