=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisService.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisService.java 2015-05-03 12:08:13 +0000 @@ -29,6 +29,7 @@ */ import java.util.Collection; +import java.util.Date; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.datavalue.DeflatedDataValue; @@ -45,5 +46,5 @@ final int MAX_OUTLIERS = 500; Collection analyse( Collection organisationUnits, Collection dataElements, - Collection periods, Double stdDevFactor ); + Collection periods, Double stdDevFactor, Date from ); } === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisStore.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/DataAnalysisStore.java 2015-05-03 12:08:13 +0000 @@ -29,6 +29,7 @@ */ import java.util.Collection; +import java.util.Date; import java.util.Map; import java.util.Set; @@ -51,10 +52,11 @@ * * @param dataElement the DataElement. * @param categoryOptionCombo the DataElementCategoryOptionCombo. - * @param organisationUnit the OrganisationUnit. + * @param organisationUnits the set of OrganisationUnit identifiers. + * @param from the from date for which to include data values. * @return a mapping between organisation unit identifier and its standard deviation. */ - Map getStandardDeviation( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits ); + Map getStandardDeviation( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits, Date from ); /** * Calculates the average of the DataValues registered for the given @@ -62,10 +64,11 @@ * * @param dataElement the DataElement. * @param categoryOptionCombo the DataElementCategoryOptionCombo. - * @param organisationUnit the OrganisationUnit. + * @param organisationUnits the set of OrganisationUnit identifiers. + * @param from the from date for which to include data values. * @return a mapping between organisation unit identifier and its average data value. */ - Map getAverage( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits ); + Map getAverage( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits, Date from ); /** * Generates a collection of data value violations of min-max predefined values. === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/MinMaxDataAnalysisService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/MinMaxDataAnalysisService.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataanalysis/MinMaxDataAnalysisService.java 2015-05-03 12:08:13 +0000 @@ -29,6 +29,7 @@ */ import java.util.Collection; +import java.util.Date; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -37,5 +38,5 @@ extends DataAnalysisService { void generateMinMaxValues( Collection organisationUnits, - Collection dataElements, Double stdDevFactor ); + Collection dataElements, Double stdDevFactor, Date from ); } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisService.java 2015-04-16 11:14:54 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisService.java 2015-05-03 12:08:13 +0000 @@ -29,6 +29,7 @@ */ import java.util.Collection; +import java.util.Date; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -94,7 +95,7 @@ @Override public Collection analyse( Collection organisationUnits, - Collection dataElements, Collection periods, Double stdDevFactor ) + Collection dataElements, Collection periods, Double stdDevFactor, Date from ) { Set elements = new HashSet<>( dataElements ); @@ -114,7 +115,7 @@ @Override public void generateMinMaxValues( Collection organisationUnits, - Collection dataElements, Double stdDevFactor ) + Collection dataElements, Double stdDevFactor, Date from ) { log.info( "Starting min-max value generation, no of data elements: " + dataElements.size() + ", no of org units: " + organisationUnits.size() ); @@ -134,9 +135,9 @@ for ( DataElementCategoryOptionCombo categoryOptionCombo : categoryOptionCombos ) { - Map standardDeviations = dataAnalysisStore.getStandardDeviation( dataElement, categoryOptionCombo, orgUnitIds ); + Map standardDeviations = dataAnalysisStore.getStandardDeviation( dataElement, categoryOptionCombo, orgUnitIds, from ); - Map averages = dataAnalysisStore.getAverage( dataElement, categoryOptionCombo, standardDeviations.keySet() ); + Map averages = dataAnalysisStore.getAverage( dataElement, categoryOptionCombo, standardDeviations.keySet(), from ); for ( Integer unit : averages.keySet() ) { === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisService.java 2015-04-29 10:29:05 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisService.java 2015-05-03 12:08:13 +0000 @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -70,12 +71,12 @@ @Override public final Collection analyse( Collection organisationUnits, - Collection dataElements, Collection periods, Double stdDevFactor ) + Collection dataElements, Collection periods, Double stdDevFactor, Date from ) { + log.info( "Starting std dev analysis, no of org units: " + organisationUnits.size() + ", factor: " + stdDevFactor + ", from: " + from ); + Set units = new HashSet<>( ConversionUtils.getIdentifiers( OrganisationUnit.class, organisationUnits ) ); - log.info( "Starting std dev analysis, no of org units: " + organisationUnits.size() + ", factor: " + stdDevFactor ); - Collection outlierCollection = new ArrayList<>(); loop : for ( DataElement dataElement : dataElements ) @@ -90,9 +91,9 @@ for ( DataElementCategoryOptionCombo categoryOptionCombo : categoryOptionCombos ) { - Map standardDeviations = dataAnalysisStore.getStandardDeviation( dataElement, categoryOptionCombo, units ); + Map standardDeviations = dataAnalysisStore.getStandardDeviation( dataElement, categoryOptionCombo, units, from ); - Map averages = dataAnalysisStore.getAverage( dataElement, categoryOptionCombo, standardDeviations.keySet() ); + Map averages = dataAnalysisStore.getAverage( dataElement, categoryOptionCombo, standardDeviations.keySet(), from ); Map lowBoundMap = new HashMap<>(); Map highBoundMap = new HashMap<>(); @@ -119,7 +120,7 @@ } } } - + return outlierCollection; } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataanalysis/jdbc/JdbcDataAnalysisStore.java 2015-05-03 12:08:13 +0000 @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,6 +52,7 @@ import org.hisp.dhis.period.Period; import org.hisp.dhis.system.objectmapper.DeflatedDataValueNameMinMaxRowMapper; import org.hisp.dhis.system.util.ConversionUtils; +import org.hisp.dhis.system.util.DateUtils; import org.hisp.dhis.system.util.PaginatedList; import org.hisp.dhis.system.util.TextUtils; import org.springframework.jdbc.core.JdbcTemplate; @@ -88,7 +90,7 @@ // ------------------------------------------------------------------------- @Override - public Map getStandardDeviation( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits ) + public Map getStandardDeviation( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits, Date from ) { Map map = new HashMap<>(); @@ -99,10 +101,13 @@ final String sql = "select ou.organisationunitid, " + - "(select stddev_pop( cast( value as " + statementBuilder.getDoubleColumnType() + " ) ) " + - "from datavalue where dataelementid = " + dataElement.getId() + " " + - "and categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + - "and sourceid = ou.organisationunitid) as deviation " + + "(select stddev_pop( cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) ) " + + "from datavalue dv " + + "inner join period pe on dv.periodid = pe.periodid " + + "where dv.dataelementid = " + dataElement.getId() + " " + + "and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + + "and pe.startdate >= '" + DateUtils.getMediumDateString( from ) + "' " + + "and dv.sourceid = ou.organisationunitid) as deviation " + "from organisationunit ou " + "where ou.organisationunitid in (" + getCommaDelimitedString( organisationUnits ) + ")"; @@ -122,7 +127,7 @@ } @Override - public Map getAverage( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits ) + public Map getAverage( DataElement dataElement, DataElementCategoryOptionCombo categoryOptionCombo, Set organisationUnits, Date from ) { Map map = new HashMap<>(); @@ -133,10 +138,13 @@ final String sql = "select ou.organisationunitid, " + - "(select avg( cast( value as " + statementBuilder.getDoubleColumnType() + " ) ) " + - "from datavalue where dataelementid = " + dataElement.getId() + " " + - "and categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + - "and sourceid = ou.organisationunitid) as average " + + "(select avg( cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) ) " + + "from datavalue dv " + + "inner join period pe on dv.periodid = pe.periodid " + + "where dv.dataelementid = " + dataElement.getId() + " " + + "and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + + "and pe.startdate >= '" + DateUtils.getMediumDateString( from ) + "' " + + "and dv.sourceid = ou.organisationunitid) as average " + "from organisationunit ou " + "where ou.organisationunitid in (" + getCommaDelimitedString( organisationUnits ) + ")"; === modified file 'dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java' --- dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java 2015-05-03 12:08:13 +0000 @@ -31,6 +31,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -94,6 +95,8 @@ private Period periodH; private Period periodI; private Period periodJ; + + private Date from = getDate( 1998, 1, 1 ); private OrganisationUnit organisationUnitA; private OrganisationUnit organisationUnitB; @@ -157,8 +160,8 @@ dataValueService.addDataValue( createDataValue( dataElementA, periodI, organisationUnitA, "3", categoryOptionCombo ) ); dataValueService.addDataValue( createDataValue( dataElementA, periodJ, organisationUnitA, "15", categoryOptionCombo ) ); - assertEquals( 15.26, dataAnalysisStore.getStandardDeviation( dataElementA, categoryOptionCombo, organisationUnits ).get( organisationUnitA.getId() ), DELTA ); - assertNull( dataAnalysisStore.getStandardDeviation( dataElementA, categoryOptionCombo, organisationUnits ).get( organisationUnitB.getId() ) ); + assertEquals( 15.26, dataAnalysisStore.getStandardDeviation( dataElementA, categoryOptionCombo, organisationUnits, from ).get( organisationUnitA.getId() ), DELTA ); + assertNull( dataAnalysisStore.getStandardDeviation( dataElementA, categoryOptionCombo, organisationUnits, from ).get( organisationUnitB.getId() ) ); } @Test @@ -175,7 +178,7 @@ dataValueService.addDataValue( createDataValue( dataElementA, periodI, organisationUnitA, "3", categoryOptionCombo ) ); dataValueService.addDataValue( createDataValue( dataElementA, periodJ, organisationUnitA, "15", categoryOptionCombo ) ); - assertEquals( 12.78, dataAnalysisStore.getAverage( dataElementA, categoryOptionCombo, organisationUnits ).get( organisationUnitA.getId() ), DELTA ); - assertNull( dataAnalysisStore.getAverage( dataElementA, categoryOptionCombo, organisationUnits ).get( organisationUnitB.getId() ) ); + assertEquals( 12.78, dataAnalysisStore.getAverage( dataElementA, categoryOptionCombo, organisationUnits, from ).get( organisationUnitA.getId() ), DELTA ); + assertNull( dataAnalysisStore.getAverage( dataElementA, categoryOptionCombo, organisationUnits, from ).get( organisationUnitB.getId() ) ); } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisServiceTest.java' --- dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisServiceTest.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/MinMaxOutlierAnalysisServiceTest.java 2015-05-03 12:08:13 +0000 @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -116,6 +117,8 @@ private Period periodI; private Period periodJ; + private Date from = getDate( 1998, 1, 1 ); + private OrganisationUnit organisationUnitA; private MinMaxDataElement minMaxDataElement; @@ -195,7 +198,7 @@ periods.add( periodA ); periods.add( periodE ); - Collection result = minMaxOutlierAnalysisService.analyse( ListUtils.getCollection( organisationUnitA ), dataElementsA, periods, null ); + Collection result = minMaxOutlierAnalysisService.analyse( ListUtils.getCollection( organisationUnitA ), dataElementsA, periods, null, from ); assertEquals( 2, result.size() ); } === modified file 'dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisServiceTest.java' --- dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisServiceTest.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/dataanalysis/StdDevOutlierAnalysisServiceTest.java 2015-05-03 12:08:13 +0000 @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -123,6 +124,8 @@ private Period periodJ; + private Date from = getDate( 1998, 1, 1 ); + private OrganisationUnit organisationUnitA; // ---------------------------------------------------------------------- @@ -202,7 +205,7 @@ periods.add( periodE ); Collection values = stdDevOutlierAnalysisService.analyse( - ListUtils.getCollection( organisationUnitA ), dataElementsA, periods, stdDevFactor ); + ListUtils.getCollection( organisationUnitA ), dataElementsA, periods, stdDevFactor, from ); double lowerBound = -34.51 * stdDevFactor; double upperBound = 34.51 * stdDevFactor; === modified file 'dhis-2/dhis-web/dhis-web-dataentry/src/main/java/org/hisp/dhis/de/action/ValidationAction.java' --- dhis-2/dhis-web/dhis-web-dataentry/src/main/java/org/hisp/dhis/de/action/ValidationAction.java 2015-02-09 17:46:40 +0000 +++ dhis-2/dhis-web/dhis-web-dataentry/src/main/java/org/hisp/dhis/de/action/ValidationAction.java 2015-05-03 12:08:13 +0000 @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -53,6 +54,7 @@ import org.hisp.dhis.validation.ValidationResult; import org.hisp.dhis.validation.ValidationRuleService; import org.hisp.dhis.webapi.utils.InputUtils; +import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import com.opensymphony.xwork2.Action; @@ -231,10 +233,12 @@ Collections.sort( organisationUnits, IdentifiableObjectNameComparator.INSTANCE ); + Date from = new DateTime( period.getStartDate() ).minusYears( 2 ).toDate(); + for ( OrganisationUnit organisationUnit : organisationUnits ) { List values = new ArrayList<>( minMaxOutlierAnalysisService.analyse( getCollection( organisationUnit ), - dataSet.getDataElements(), getCollection( period ), null ) ); + dataSet.getDataElements(), getCollection( period ), null, from ) ); if ( !values.isEmpty() ) { === modified file 'dhis-2/dhis-web/dhis-web-light/src/main/java/org/hisp/dhis/light/utils/FormUtilsImpl.java' --- dhis-2/dhis-web/dhis-web-light/src/main/java/org/hisp/dhis/light/utils/FormUtilsImpl.java 2015-02-19 09:18:17 +0000 +++ dhis-2/dhis-web/dhis-web-light/src/main/java/org/hisp/dhis/light/utils/FormUtilsImpl.java 2015-05-03 12:08:13 +0000 @@ -56,6 +56,7 @@ import org.hisp.dhis.validation.ValidationResult; import org.hisp.dhis.validation.ValidationRule; import org.hisp.dhis.validation.ValidationRuleService; +import org.joda.time.DateTime; import java.util.ArrayList; import java.util.Collection; @@ -154,11 +155,13 @@ Double factor = (Double) systemSettingManager.getSystemSetting( SystemSettingManager.KEY_FACTOR_OF_DEVIATION, 2.0 ); + Date from = new DateTime( period.getStartDate() ).minusYears( 2 ).toDate(); + Collection stdDevs = stdDevOutlierAnalysisService.analyse( - ListUtils.getCollection( organisationUnit ), dataElements, ListUtils.getCollection( period ), factor ); + ListUtils.getCollection( organisationUnit ), dataElements, ListUtils.getCollection( period ), factor, from ); Collection minMaxs = minMaxOutlierAnalysisService.analyse( - ListUtils.getCollection( organisationUnit ), dataElements, ListUtils.getCollection( period ), null ); + ListUtils.getCollection( organisationUnit ), dataElements, ListUtils.getCollection( period ), null, from ); Collection deflatedDataValues = CollectionUtils.union( stdDevs, minMaxs ); === modified file 'dhis-2/dhis-web/dhis-web-validationrule/src/main/java/org/hisp/dhis/validationrule/action/dataanalysis/GetAnalysisAction.java' --- dhis-2/dhis-web/dhis-web-validationrule/src/main/java/org/hisp/dhis/validationrule/action/dataanalysis/GetAnalysisAction.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-web/dhis-web-validationrule/src/main/java/org/hisp/dhis/validationrule/action/dataanalysis/GetAnalysisAction.java 2015-05-03 12:08:13 +0000 @@ -29,6 +29,7 @@ */ import com.opensymphony.xwork2.Action; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.ServiceProvider; @@ -43,10 +44,13 @@ import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.util.SessionUtils; +import org.joda.time.DateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -139,9 +143,9 @@ this.fromDate = fromDate.trim(); } - private Collection dataSets; + private List dataSets = new ArrayList<>(); - public void setDataSets( Collection dataSets ) + public void setDataSets( List dataSets ) { this.dataSets = dataSets; } @@ -185,35 +189,36 @@ @Override public String execute() { + OrganisationUnit unit = selectionTreeManager.getReloadedSelectedOrganisationUnit(); + + if ( fromDate == null || toDate == null || dataSets == null || unit == null ) + { + return ERROR; + } + + Collection orgUnits = organisationUnitService.getOrganisationUnitWithChildren( unit.getId() ); + + Collection periods = periodService.getPeriodsBetweenDates( format.parseDate( fromDate ), format.parseDate( toDate ) ); + Set dataElements = new HashSet<>(); - Collection periods = null; - OrganisationUnit unit = selectionTreeManager.getReloadedSelectedOrganisationUnit(); - Collection orgUnits = null; - - // TODO filter periods with data element period type - - if ( fromDate != null && toDate != null && dataSets != null && unit != null ) + + for ( String uid : dataSets ) { - orgUnits = organisationUnitService.getOrganisationUnitWithChildren( unit.getId() ); - - periods = periodService.getPeriodsBetweenDates( format.parseDate( fromDate ), format.parseDate( toDate ) ); - - for ( String uid : dataSets ) - { - dataElements.addAll( dataSetService.getDataSet( uid ).getDataElements() ); - } - - log.info( "From date: " + fromDate + ", To date: " + toDate + ", Organisation unit: " + unit - + ", Std dev: " + standardDeviation + ", Key: " + key ); - - log.info( "Nr of data elements: " + dataElements.size() + " Nr of periods: " + periods.size() ); + dataElements.addAll( dataSetService.getDataSet( uid ).getDataElements() ); } + Date from = new DateTime( fromDate ).minusYears( 2 ).toDate(); + + log.info( "From date: " + fromDate + ", To date: " + toDate + ", Organisation unit: " + unit + + ", Std dev: " + standardDeviation + ", Key: " + key ); + + log.info( "Nr of data elements: " + dataElements.size() + " Nr of periods: " + periods.size() ); + DataAnalysisService service = serviceProvider.provide( key ); if ( service != null ) { - dataValues = service.analyse( orgUnits, dataElements, periods, standardDeviation ); + dataValues = service.analyse( orgUnits, dataElements, periods, standardDeviation, from ); maxExceeded = dataValues.size() > DataAnalysisService.MAX_OUTLIERS; }