=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/Expression.java 2014-07-16 15:29:27 +0000 @@ -28,12 +28,14 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; + import org.apache.commons.lang.Validate; import org.hisp.dhis.common.BaseIdentifiableObject; import org.hisp.dhis.common.DxfNamespaces; @@ -113,6 +115,12 @@ private Set optionCombosInExpression = new HashSet(); // ------------------------------------------------------------------------- + // Transient properties + // ------------------------------------------------------------------------- + + private transient String explodedExpression; + + // ------------------------------------------------------------------------- // Constructors // ------------------------------------------------------------------------- @@ -140,6 +148,18 @@ } // ------------------------------------------------------------------------- + // Logic + // ------------------------------------------------------------------------- + + /** + * Returns exploded expression, if null returns expression. + */ + public String getExplodedExpressionFallback() + { + return explodedExpression != null ? explodedExpression : expression; + } + + // ------------------------------------------------------------------------- // Equals and hashCode // ------------------------------------------------------------------------- @@ -195,8 +215,8 @@ { final int PRIME = 31; int result = 1; - result = PRIME * result + ((description == null) ? 0 : description.hashCode()); - result = PRIME * result + ((expression == null) ? 0 : expression.hashCode()); + result = PRIME * result + ( ( description == null ) ? 0 : description.hashCode() ); + result = PRIME * result + ( ( expression == null ) ? 0 : expression.hashCode() ); return result; } @@ -207,6 +227,7 @@ return "Expression{" + "id=" + id + ", expression='" + expression + '\'' + + ", explodedExpression='" + explodedExpression + '\'' + ", description='" + description + '\'' + ", dataElementsInExpression=" + dataElementsInExpression.size() + ", optionCombosInExpression=" + optionCombosInExpression.size() + @@ -296,6 +317,17 @@ this.nullIfBlank = nullIfBlank; } + @JsonIgnore + public String getExplodedExpression() + { + return explodedExpression; + } + + public void setExplodedExpression( String explodedExpression ) + { + this.explodedExpression = explodedExpression; + } + public void mergeWith( Expression other ) { Validate.notNull( other ); === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ExpressionService.java 2014-07-16 15:29:27 +0000 @@ -39,6 +39,7 @@ import org.hisp.dhis.indicator.Indicator; import org.hisp.dhis.organisationunit.OrganisationUnitGroup; import org.hisp.dhis.period.Period; +import org.hisp.dhis.validation.ValidationRule; /** * Expressions are mathematical formulas and can contain references to various @@ -277,6 +278,9 @@ * Populates the explodedNumerator and explodedDenominator property on all * indicators in the given collection. This method uses * explodeExpression( String ) internally to generate the exploded expressions. + * Replaces references to data element totals with references to all + * category option combos in the category combo for that data element. + * * This method will perform better compared to calling explodeExpression( String ) * multiple times outside a transactional context as the transactional * overhead is avoided. @@ -290,12 +294,25 @@ * Populates the explodedNumerator and explodedDenominator property on all * indicators in the given collection. This method uses * explodeExpression( String ) internally to generate the exploded expressions. + * Replaces references to data element totals with references to all + * category option combos in the category combo for that data element. * * @param indicators the collection of indicators. */ void explodeExpressions( Collection indicators ); /** + * Populates the explodedExpression property on the Expression object of all + * validation rules in the given collection. This method uses + * explodeExpression( String ) internally to generate the exploded expressions. + * Replaces references to data element totals with references to all + * category option combos in the category combo for that data element. + * + * @param validationRules the collection of validation rules. + */ + void explodeValidationRuleExpressions( Collection validationRules ); + + /** * Replaces references to data element totals with references to all * category option combos in the category combo for that data element. * === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java 2014-04-25 11:22:12 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/indicator/Indicator.java 2014-07-16 15:29:27 +0000 @@ -41,6 +41,7 @@ import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.mapping.MapLegendSet; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -68,13 +69,13 @@ private String numeratorDescription; - private String explodedNumerator; + private transient String explodedNumerator; private String denominator; private String denominatorDescription; - private String explodedDenominator; + private transient String explodedDenominator; private Integer sortOrder; @@ -222,9 +223,7 @@ this.numeratorDescription = numeratorDescription; } - @JsonProperty - @JsonView( {DetailedView.class, ExportView.class} ) - @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0) + @JsonIgnore public String getExplodedNumerator() { return explodedNumerator; @@ -261,9 +260,7 @@ this.denominatorDescription = denominatorDescription; } - @JsonProperty - @JsonView( {DetailedView.class, ExportView.class} ) - @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0) + @JsonIgnore public String getExplodedDenominator() { return explodedDenominator; === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2014-05-18 00:49:40 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/validation/ValidationResult.java 2014-07-16 15:29:27 +0000 @@ -95,11 +95,11 @@ return result; } - // - // Note: this method is called from threads in which it may not be possible - // to initialize lazy Hibernate properties. So object properties to compare - // must be chosen accordingly. - // + /** + * Note: this method is called from threads in which it may not be possible + * to initialize lazy Hibernate properties. So object properties to compare + * must be chosen accordingly. + */ @Override public boolean equals( Object object ) { @@ -203,11 +203,11 @@ return true; } - // - // Note: this method is called from threads in which it may not be possible - // to initialize lazy Hibernate properties. So object properties to compare - // must be chosen accordingly. - // + /** + * Note: this method is called from threads in which it may not be possible + * to initialize lazy Hibernate properties. So object properties to compare + * must be chosen accordingly. + */ public int compareTo( ValidationResult other ) { int result = source.getName().compareTo( other.source.getName() ); @@ -291,7 +291,11 @@ @Override public String toString() { - return source + " - " + period + " - " + attributeOptionCombo.getName() + " - " + validationRule + " - " + leftsideValue + " - " + rightsideValue; + return "[Source: " + source + + ", period: " + period + + ", validation rule: " + validationRule + + ", left side value: " + leftsideValue + + ", right side value: " + rightsideValue + "]"; } // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/DefaultExpressionService.java 2014-07-16 15:29:27 +0000 @@ -63,6 +63,7 @@ import org.hisp.dhis.period.Period; import org.hisp.dhis.system.util.DateUtils; import org.hisp.dhis.system.util.MathUtils; +import org.hisp.dhis.validation.ValidationRule; import org.springframework.transaction.annotation.Transactional; /** @@ -196,7 +197,7 @@ public Double getExpressionValue( Expression expression, Map valueMap, Map constantMap, Map orgUnitCountMap, Integer days ) { - final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, + final String expressionString = generateExpression( expression.getExplodedExpressionFallback(), valueMap, constantMap, orgUnitCountMap, days, expression.isNullIfBlank() ); return expressionString != null ? calculateExpression( expressionString ) : null; @@ -205,7 +206,7 @@ public Double getExpressionValue( Expression expression, Map valueMap, Map constantMap, Map orgUnitCountMap, Integer days, Set incompleteValues ) { - final String expressionString = generateExpression( expression.getExpression(), valueMap, constantMap, orgUnitCountMap, days, + final String expressionString = generateExpression( expression.getExplodedExpressionFallback(), valueMap, constantMap, orgUnitCountMap, days, expression.isNullIfBlank(), incompleteValues ); return expressionString != null ? calculateExpression( expressionString ) : null; @@ -603,7 +604,7 @@ explodeExpressions( indicators ); } } - + @Transactional public void explodeExpressions( Collection indicators ) { @@ -632,7 +633,36 @@ } } } - + + @Transactional + public void explodeValidationRuleExpressions( Collection validationRules ) + { + if ( validationRules != null && !validationRules.isEmpty() ) + { + Set dataElementTotals = new HashSet(); + + for ( ValidationRule rule : validationRules ) + { + dataElementTotals.addAll( getDataElementTotalUids( rule.getLeftSide().getExpression() ) ); + dataElementTotals.addAll( getDataElementTotalUids( rule.getRightSide().getExpression() ) ); + } + + if ( !dataElementTotals.isEmpty() ) + { + final ListMap dataElementMap = dataElementService.getDataElementCategoryOptionComboMap( dataElementTotals ); + + if ( !dataElementMap.isEmpty() ) + { + for ( ValidationRule rule : validationRules ) + { + rule.getLeftSide().setExplodedExpression( explodeExpression( rule.getLeftSide().getExplodedExpressionFallback(), dataElementMap ) ); + rule.getRightSide().setExplodedExpression( explodeExpression( rule.getRightSide().getExplodedExpressionFallback(), dataElementMap ) ); + } + } + } + } + } + private String explodeExpression( String expression, ListMap dataElementOptionComboMap ) { if ( expression == null || expression.isEmpty() ) @@ -655,7 +685,7 @@ for ( String coc : cocs ) { - replace.append( EXP_OPEN ).append( matcher.group( 1 ) ).append( SEPARATOR ).append( + replace.append( EXP_OPEN ).append( de ).append( SEPARATOR ).append( coc ).append( EXP_CLOSE ).append( "+" ); } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2014-07-16 12:15:52 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/validation/ValidatorThread.java 2014-07-16 15:29:27 +0000 @@ -92,7 +92,8 @@ for ( PeriodTypeExtended periodTypeX : context.getPeriodTypeExtendedMap().values() ) { Collection sourceDataElements = periodTypeX.getSourceDataElements().get( sourceX.getSource() ); - Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements ); + Set rules = getRulesBySourceAndPeriodType( sourceX, periodTypeX, sourceDataElements ); + context.getExpressionService().explodeValidationRuleExpressions( rules ); if ( !rules.isEmpty() ) { @@ -112,7 +113,7 @@ for ( ValidationRule rule : rules ) { - if ( evaluateCheck( currentValueMap, lastUpdatedMap, rule ) ) + if ( evaluateValidationCheck( currentValueMap, lastUpdatedMap, rule ) ) { Map leftSideValues = getExpressionValueMap( rule.getLeftSide(), currentValueMap, incompleteValuesMap ); @@ -120,19 +121,19 @@ if ( !leftSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) ) { Map rightSideValues = getRightSideValue( sourceX.getSource(), periodTypeX, period, rule, - currentValueMap, sourceDataElements ); + currentValueMap, sourceDataElements ); if ( !rightSideValues.isEmpty() || Operator.compulsory_pair.equals( rule.getOperator() ) ) { - Set combos = leftSideValues.keySet(); + Set attributeOptionCombos = leftSideValues.keySet(); if ( Operator.compulsory_pair.equals( rule.getOperator() ) ) { - combos = new HashSet( combos ); - combos.addAll( rightSideValues.keySet() ); + attributeOptionCombos = new HashSet( attributeOptionCombos ); + attributeOptionCombos.addAll( rightSideValues.keySet() ); } - for ( int combo : combos ) + for ( int combo : attributeOptionCombos ) { Double leftSide = leftSideValues.get ( combo ); Double rightSide = rightSideValues.get ( combo ); @@ -197,6 +198,7 @@ // But if this is some funny kind of rule with no elements // (like for testing), include it also. Collection elements = rule.getCurrentDataElements(); + if ( elements == null || elements.size() == 0 || sourceDataElements.containsAll( elements ) ) { periodTypeRules.add( rule ); @@ -233,7 +235,7 @@ * @param rule the rule that may be evaluated * @return true if the rule should be evaluated with this data, false if not */ - private boolean evaluateCheck( MapMap currentValueMapMap, + private boolean evaluateValidationCheck( MapMap currentValueMapMap, MapMap lastUpdatedMapMap, ValidationRule rule ) { boolean evaluate = true; // Assume true for now. @@ -268,6 +270,7 @@ for ( DataElementOperand deo : deos ) { Date lastUpdated = entry.getValue().get( deo ); + if ( lastUpdated != null && lastUpdated.after( context.getLastScheduledRun() ) ) { saveThisCombo = true; // True if new/updated data. @@ -454,17 +457,17 @@ * @return map of values. */ private Map getExpressionValueMap( Expression expression, - MapMap valueMap, - SetMap incompleteValuesMap ) + MapMap valueMap, + SetMap incompleteValuesMap ) { Map expressionValueMap = new HashMap(); - for ( Map.Entry> e : valueMap.entrySet() ) + for ( Map.Entry> entry : valueMap.entrySet() ) { - expressionValueMap.put( e.getKey(), + expressionValueMap.put( entry.getKey(), context.getExpressionService().getExpressionValue( expression, - e.getValue(), context.getConstantMap(), null, null, - incompleteValuesMap.getSet( e.getKey() ) ) ); + entry.getValue(), context.getConstantMap(), null, null, + incompleteValuesMap.getSet( entry.getKey() ) ) ); } return expressionValueMap; @@ -480,8 +483,8 @@ * @param sequentialSampleCount number of sequential samples tried for * @return average right-side sample value */ - private Double rightSideAverage( ValidationRule rule, List sampleValues, int annualSampleCount, - int sequentialSampleCount ) + private Double rightSideAverage( ValidationRule rule, List sampleValues, + int annualSampleCount, int sequentialSampleCount ) { // Find the expected sample count for the last period of its type in the // database: sequentialSampleCount for the immediately preceding periods @@ -546,10 +549,10 @@ * @return map of attribute option combo to map of values found. */ private MapMap getValueMap( PeriodTypeExtended periodTypeX, - Collection ruleDataElements, Collection sourceDataElements, - Set recursiveDataElements, Collection allowedPeriodTypes, Period period, - OrganisationUnit source, MapMap lastUpdatedMap, - SetMap incompleteValuesMap ) + Collection ruleDataElements, Collection sourceDataElements, + Set recursiveDataElements, Collection allowedPeriodTypes, Period period, + OrganisationUnit source, MapMap lastUpdatedMap, + SetMap incompleteValuesMap ) { Set dataElementsToGet = new HashSet( ruleDataElements ); dataElementsToGet.retainAll( sourceDataElements ); === modified file 'dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js' --- dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js 2013-10-16 12:39:47 +0000 +++ dhis-2/dhis-web/dhis-web-validationrule/src/main/webapp/dhis-web-validationrule/javascript/expressionBuilder.js 2014-07-16 15:29:27 +0000 @@ -73,7 +73,7 @@ var periodTypeAllowAverage = ( ruleType && ruleType == "surveillance" ) ? true : false; dataDictionary.loadOperands( "#expression-container select[id=dataElementId]", - {usePaging: true, key: key, periodType: periodType, periodTypeAllowAverage: periodTypeAllowAverage } ); + {usePaging: true, key: key, periodType: periodType, includeTotals: true, periodTypeAllowAverage: periodTypeAllowAverage } ); } function clearSearchText()