=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java 2011-12-10 17:55:41 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java 2011-12-11 01:24:32 +0000 @@ -27,9 +27,19 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import java.util.Collection; + public interface AggregatedOrgUnitDataValueService { + void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ); + + void deleteAggregatedDataValues( Collection periodIds ); + void createIndex( boolean dataElement, boolean indicator ); void dropIndex( boolean dataElement, boolean indicator ); + + void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ); + + void deleteAggregatedIndicatorValues( Collection periodIds ); } === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java 2011-12-10 17:55:41 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java 2011-12-11 01:24:32 +0000 @@ -27,10 +27,19 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import java.util.Collection; + public interface AggregatedOrgUnitDataValueStore { + void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ); + + void deleteAggregatedDataValues( Collection periodIds ); + void createIndex( boolean dataElement, boolean indicator ); void dropIndex( boolean dataElement, boolean indicator ); - + + void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ); + + void deleteAggregatedIndicatorValues( Collection periodIds ); } === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/options/SystemSettingManager.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/options/SystemSettingManager.java 2011-11-25 18:01:49 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/options/SystemSettingManager.java 2011-12-11 15:48:13 +0000 @@ -72,7 +72,7 @@ final int DEFAULT_MAX_NUMBER_OF_ATTEMPTS = 20; final int DEFAULT_TIMEFRAME_MINUTES = 1; final double DEFAULT_FACTOR_OF_DEVIATION = 2.0; - final int DEFAULT_ORGUNITGROUPSET_AGG_LEVEL = 3; + final int DEFAULT_ORGUNITGROUPSET_AGG_LEVEL = 1; final String DEFAULT_GOOGLE_MAPS_API_KEY = "ABQIAAAAut6AhySExnYIXm5s2OFIkxRKNzJ-_9njnryRTbvC6CtrS4sRvRREWnxwlZUa630pLuPf3nD9i4fq9w"; final String AGGREGATION_STRATEGY_REAL_TIME = "real_time"; === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java 2011-12-09 20:53:07 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java 2011-12-11 01:24:32 +0000 @@ -443,9 +443,25 @@ return set; } + public int getOrganisationUnitLevel() + { + int currentLevel = 1; + + OrganisationUnit thisParent = this.parent; + + while ( thisParent != null ) + { + ++currentLevel; + + thisParent = thisParent.getParent(); + } + + return currentLevel; + } + public boolean isPolygon() { - return (featureType.equals( FEATURETYPE_MULTIPOLYGON ) || featureType.equals( FEATURETYPE_POLYGON )); + return featureType.equals( FEATURETYPE_MULTIPOLYGON ) || featureType.equals( FEATURETYPE_POLYGON ); } public boolean isPoint() === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java 2011-12-09 20:53:07 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java 2011-12-11 13:47:04 +0000 @@ -44,7 +44,8 @@ */ @XmlRootElement( name = "organisationUnitGroup", namespace = Dxf2Namespace.NAMESPACE ) @XmlAccessorType( value = XmlAccessType.NONE ) -public class OrganisationUnitGroup extends BaseIdentifiableObject +public class OrganisationUnitGroup + extends BaseIdentifiableObject { /** * Determines if a de-serialized file is compatible with this class. === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupService.java 2011-11-22 16:17:49 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupService.java 2011-12-11 13:47:04 +0000 @@ -105,6 +105,13 @@ * collection if no OrganisationUnitGroup exists. */ Collection getAllOrganisationUnitGroups(); + + /** + * Returns all OrganisationUnitGroups which have a OrganisationUnitGroupSet. + * + * @return a collection of OrganisationUnitGroups. + */ + Collection getOrganisationUnitGroupsWithGroupSets(); // ------------------------------------------------------------------------- // OrganisationUnitGroupSet === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupStore.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroupStore.java 2011-12-11 13:47:04 +0000 @@ -0,0 +1,41 @@ +package org.hisp.dhis.organisationunit; + +/* + * Copyright (c) 2004-2010, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; + +import org.hisp.dhis.common.GenericIdentifiableObjectStore; + +/** + * @author Lars Helge Overland + */ +public interface OrganisationUnitGroupStore + extends GenericIdentifiableObjectStore +{ + Collection getOrganisationUnitGroupsWithGroupSets(); +} === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2011-12-07 14:10:29 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java 2011-12-11 13:47:04 +0000 @@ -159,6 +159,13 @@ Collection getRootOrganisationUnits(); /** + * Returns the level of the organisation unit with the given identifier. + * + * @return the level of the organisation unit with the given identifier. + */ + int getLevelOfOrganisationUnit( int id ); + + /** * Returns all OrganisationUnits which are part of the subtree of the * OrganisationUnit with the given identifer and have no children. * @@ -213,24 +220,6 @@ Collection getOrganisationUnitsAtLevel( int level, OrganisationUnit parent ); /** - * Returns the hierarchical level in which the given OrganisationUnit - * resides. - * - * @param id the identifier of the OrganisationUnit. - * @return the hierarchical level of the given OrganisationUnit. - */ - int getLevelOfOrganisationUnit( int id ); - - /** - * Returns the hierarchical level in which the given OrganisationUnit - * resides. - * - * @param organisationUnit the OrganisationUnit. - * @return the hierarchical level of the given OrganisationUnit. - */ - int getLevelOfOrganisationUnit( OrganisationUnit organisationUnit ); - - /** * Returns the number of levels in the OrganisationUnit hierarchy. * * @return the number of hierarchical levels. === modified file 'dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataprune/DefaultDataPruneService.java' --- dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataprune/DefaultDataPruneService.java 2010-11-11 07:30:38 +0000 +++ dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/dataprune/DefaultDataPruneService.java 2011-12-11 13:47:04 +0000 @@ -87,14 +87,14 @@ if ( organisationUnit.getParent() != null ) { OrganisationUnitLevel level = organisationUnitService - .getOrganisationUnitLevelByLevel( organisationUnitService.getLevelOfOrganisationUnit( organisationUnit - .getParent() ) ); + .getOrganisationUnitLevelByLevel( organisationUnitService.getLevelOfOrganisationUnit( organisationUnit.getParent().getId() ) ); if ( level != null ) { organisationUnitService.deleteOrganisationUnitLevel( level ); } } + if ( organisationUnit.getParent().getParent() != null ) { deleteLevels( organisationUnit.getParent() ); === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java 2011-12-10 17:55:41 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java 2011-12-11 01:24:32 +0000 @@ -1,5 +1,7 @@ package org.hisp.dhis.aggregation; +import java.util.Collection; + /* * Copyright (c) 2004-2010, University of Oslo * All rights reserved. @@ -44,6 +46,16 @@ // ------------------------------------------------------------------------- // AggregatedOrgUnitDataValueService implementation // ------------------------------------------------------------------------- + + public void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ) + { + aggregatedDataValueStore.deleteAggregatedDataValues( dataElementIds, periodIds, organisationUnitIds ); + } + + public void deleteAggregatedDataValues( Collection periodIds ) + { + aggregatedDataValueStore.deleteAggregatedDataValues( periodIds ); + } public void createIndex( boolean dataElement, boolean indicator ) { @@ -54,4 +66,18 @@ { aggregatedDataValueStore.dropIndex( dataElement, indicator ); } + + // ------------------------------------------------------------------------- + // AggregatedOrgUnitDataValueService implementation + // ------------------------------------------------------------------------- + + public void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ) + { + aggregatedDataValueStore.deleteAggregatedIndicatorValues( indicatorIds, periodIds, organisationUnitIds ); + } + + public void deleteAggregatedIndicatorValues( Collection periodIds ) + { + aggregatedDataValueStore.deleteAggregatedDataValues( periodIds ); + } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2011-12-10 17:55:41 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2011-12-11 01:24:32 +0000 @@ -27,6 +27,10 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString; + +import java.util.Collection; + import org.amplecode.quick.StatementManager; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -60,6 +64,26 @@ // AggregatedOrgUnitDataValueStore implementation // ------------------------------------------------------------------------- + public void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitdatavalue " + + "WHERE dataelementid IN ( " + getCommaDelimitedString( dataElementIds ) + " ) " + + "AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " + + "AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )"; + + jdbcTemplate.execute( sql ); + } + + public void deleteAggregatedDataValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitdatavalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + jdbcTemplate.execute( sql ); + } + public void createIndex( boolean dataElement, boolean indicator ) { if ( dataElement ) @@ -117,4 +141,29 @@ } } } + + // ------------------------------------------------------------------------- + // AggregatedIndicatorValue + // ------------------------------------------------------------------------- + + public void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, + Collection organisationUnitIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitindicatorvalue " + + "WHERE indicatorid IN ( " + getCommaDelimitedString( indicatorIds ) + " ) " + + "AND periodid IN ( " + getCommaDelimitedString( periodIds ) + " ) " + + "AND organisationunitid IN ( " + getCommaDelimitedString( organisationUnitIds ) + " )"; + + jdbcTemplate.execute( sql ); + } + + public void deleteAggregatedIndicatorValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitindicatorvalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + jdbcTemplate.execute( sql ); + } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitGroupService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitGroupService.java 2011-11-22 16:17:49 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitGroupService.java 2011-12-11 13:47:04 +0000 @@ -47,9 +47,9 @@ // Dependencies // ------------------------------------------------------------------------- - private GenericIdentifiableObjectStore organisationUnitGroupStore; + private OrganisationUnitGroupStore organisationUnitGroupStore; - public void setOrganisationUnitGroupStore( GenericIdentifiableObjectStore organisationUnitGroupStore ) + public void setOrganisationUnitGroupStore( OrganisationUnitGroupStore organisationUnitGroupStore ) { this.organisationUnitGroupStore = organisationUnitGroupStore; } @@ -112,6 +112,11 @@ { return organisationUnitGroupStore.getAll(); } + + public Collection getOrganisationUnitGroupsWithGroupSets() + { + return organisationUnitGroupStore.getOrganisationUnitGroupsWithGroupSets(); + } // ------------------------------------------------------------------------- // OrganisationUnitGroupSet === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2011-12-09 18:30:43 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java 2011-12-11 13:47:04 +0000 @@ -206,6 +206,11 @@ { return organisationUnitStore.getRootOrganisationUnits(); } + + public int getLevelOfOrganisationUnit( int id ) + { + return getOrganisationUnit( id ).getOrganisationUnitLevel(); + } public Collection getLeafOrganisationUnits( int id ) { @@ -323,7 +328,7 @@ throw new IllegalArgumentException( "Level must be greater than zero" ); } - int parentLevel = getLevelOfOrganisationUnit( parent ); + int parentLevel = parent.getOrganisationUnitLevel(); if ( level < parentLevel ) { @@ -366,27 +371,6 @@ } } - public int getLevelOfOrganisationUnit( int id ) - { - return getLevelOfOrganisationUnit( getOrganisationUnit( id ) ); - } - - public int getLevelOfOrganisationUnit( OrganisationUnit organisationUnit ) - { - int level = 1; - - OrganisationUnit parent = organisationUnit.getParent(); - - while ( parent != null ) - { - ++level; - - parent = parent.getParent(); - } - - return level; - } - public int getNumberOfOrganisationalLevels() { int maxDepth = 0; === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitGroupStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitGroupStore.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitGroupStore.java 2011-12-11 13:47:04 +0000 @@ -0,0 +1,48 @@ +package org.hisp.dhis.organisationunit.hibernate; + +/* + * Copyright (c) 2004-2010, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; + +import org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; +import org.hisp.dhis.organisationunit.OrganisationUnitGroupStore; + +/** + * @author Lars Helge Overland + */ +public class HibernateOrganisationUnitGroupStore + extends HibernateIdentifiableObjectStore + implements OrganisationUnitGroupStore +{ + @SuppressWarnings( "unchecked" ) + public Collection getOrganisationUnitGroupsWithGroupSets() + { + return getQuery( "from OrganisationUnitGroup o where o.groupSet is not null" ).list(); + } +} === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2011-11-25 15:12:06 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/hibernate/HibernateOrganisationUnitStore.java 2011-12-11 13:47:04 +0000 @@ -37,13 +37,10 @@ import java.util.Map; import java.util.Set; -import org.amplecode.quick.StatementHolder; -import org.amplecode.quick.StatementManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.hibernate.Criteria; import org.hibernate.Query; -import org.hibernate.Session; import org.hibernate.criterion.Restrictions; import org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -57,8 +54,6 @@ /** * @author Kristian Nordal - * @version $Id: HibernateOrganisationUnitStore.java 6251 2008-11-10 14:37:05Z - * larshelg $ */ public class HibernateOrganisationUnitStore extends HibernateIdentifiableObjectStore @@ -68,20 +63,14 @@ // Dependencies // ------------------------------------------------------------------------- - private StatementManager statementManager; - - public void setStatementManager( StatementManager statementManager ) - { - this.statementManager = statementManager; - } - + //TODO this should be a separate class! + private HibernateIdentifiableObjectStore orgLevelStore; public void setOrgLevelStore( HibernateIdentifiableObjectStore orgLevelStore ) { this.orgLevelStore = orgLevelStore; - } - + } // ------------------------------------------------------------------------- // OrganisationUnit @@ -90,25 +79,19 @@ @Override public OrganisationUnit getOrganisationUnitByNameIgnoreCase( String name ) { - Criteria criteria = sessionFactory.getCurrentSession().createCriteria( OrganisationUnit.class ); - criteria.add( Restrictions.eq( "name", name ).ignoreCase() ); - return (OrganisationUnit) criteria.uniqueResult(); + return (OrganisationUnit) getCriteria( Restrictions.eq( "name", name ).ignoreCase() ).uniqueResult(); } @SuppressWarnings( "unchecked" ) public Collection getRootOrganisationUnits() { - Session session = sessionFactory.getCurrentSession(); - - return session.createQuery( "from OrganisationUnit o where o.parent is null" ).list(); + return getQuery( "from OrganisationUnit o where o.parent is null" ).list(); } @SuppressWarnings( "unchecked" ) public Collection getOrganisationUnitsWithoutGroups() { - String hql = "from OrganisationUnit o where o.groups.size = 0"; - - return sessionFactory.getCurrentSession().createQuery( hql ).list(); + return getQuery( "from OrganisationUnit o where o.groups.size = 0" ).list(); } @SuppressWarnings( "unchecked" ) @@ -227,12 +210,10 @@ { Timestamp now = new Timestamp( new Date().getTime() ); - StatementHolder holder = statementManager.getHolder(); - final String sql = "update organisationunit " + "set parentid=" + parentId + ", lastupdated='" + now + "' " + "where organisationunitid=" + organisationUnitId; - holder.executeUpdate( sql ); + jdbcTemplate.execute( sql ); } // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2011-12-10 17:55:41 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2011-12-11 13:47:04 +0000 @@ -153,13 +153,12 @@ - - + === modified file 'dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java' --- dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java 2011-11-22 15:48:30 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java 2011-12-11 01:24:32 +0000 @@ -291,9 +291,9 @@ assertTrue( organisationUnitService.getOrganisationUnitsAtLevel( 1 ).size() == 2 ); assertTrue( organisationUnitService.getOrganisationUnitsAtLevel( 3 ).size() == 3 ); assertTrue( organisationUnitService.getNumberOfOrganisationalLevels() == 4 ); - assertTrue( organisationUnitService.getLevelOfOrganisationUnit( unit4 ) == 3 ); - assertTrue( organisationUnitService.getLevelOfOrganisationUnit( unit1 ) == 1 ); - assertTrue( organisationUnitService.getLevelOfOrganisationUnit( unit6 ) == 4 ); + assertTrue( organisationUnitService.getOrganisationUnit( unit4.getId() ).getOrganisationUnitLevel() == 3 ); + assertTrue( organisationUnitService.getOrganisationUnit( unit1.getId() ).getOrganisationUnitLevel() == 1 ); + assertTrue( organisationUnitService.getOrganisationUnit( unit6.getId() ).getOrganisationUnitLevel() == 4 ); } @Test === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataElementOperandList.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataElementOperandList.java 2011-12-10 18:15:44 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataElementOperandList.java 2011-12-11 13:47:04 +0000 @@ -67,7 +67,7 @@ this.valueList[0] = period.getId(); this.valueList[1] = unit.getId(); - if ( group != null ) + if ( group != null && group.getId() != 0 ) { this.valueList[2] = group.getId(); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-10-29 14:16:54 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-12-11 16:07:13 +0000 @@ -35,6 +35,9 @@ import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.datamart.CrossTabDataValue; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; +import org.hisp.dhis.period.Period; /** * @author Lars Helge Overland @@ -86,6 +89,24 @@ * @param key the key used in the table name. */ void dropAggregatedDataCache( String key ); + + /** + * Creates a table which functions as a cache for aggregated org unit data + * element values with columns for period identifier, organisation unit + * identifier, organisation unit group identifier followed by one column for + * each DataElementOperand in the given list. + * + * @param operands the list of DataElementOperands. + * @param key the key to use in table name. + */ + void createAggregatedOrgUnitDataCache( List operands, String key ); + + /** + * Drops the aggregated org unit data cache table. + * + * @param key the key used in the table name. + */ + void dropAggregatedOrgUnitDataCache( String key ); /** * Gets all CrossTabDataValues for the given collection of period ids and source ids. @@ -111,14 +132,17 @@ /** * Gets a map of DataElementOperands and corresponding Double aggregated data - * element value from the cache table. + * element value from the cache table. If the group argument is not null it + * will read from the aggregated org unit data cache, if null it will read from + * the aggregated data cache. * * @param operands the list of DataElementOperand to return map entries for. - * @param periodId the period identifier. - * @param sourceId the organisation unit identifier. + * @param period the Period. + * @param unit the OrganisationUnit. + * @param group the OrganisationUnitGroup. * @param key the key to use in the table name. * @return a map of DataElementOperands and aggregated values. */ Map getAggregatedDataCacheValue( Collection operands, - int periodId, int sourceId, String key ); + Period period, OrganisationUnit unit, OrganisationUnitGroup group, String key ); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-10-29 14:16:54 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2011-12-11 16:07:13 +0000 @@ -49,6 +49,9 @@ import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; +import org.hisp.dhis.period.Period; import org.springframework.scheduling.annotation.Async; /** @@ -201,6 +204,16 @@ crossTabStore.dropAggregatedDataCache( key ); } + public void createAggregatedOrgUnitDataCache( List operands, String key ) + { + crossTabStore.createAggregatedOrgUnitDataCache( operands, key ); + } + + public void dropAggregatedOrgUnitDataCache( String key ) + { + crossTabStore.dropAggregatedOrgUnitDataCache( key ); + } + public Collection getCrossTabDataValues( Collection operands, Collection periodIds, Collection sourceIds, String key ) { @@ -214,8 +227,15 @@ } public Map getAggregatedDataCacheValue( Collection operands, - int periodId, int sourceId, String key ) + Period period, OrganisationUnit unit, OrganisationUnitGroup group, String key ) { - return crossTabStore.getAggregatedDataCacheValue( operands, periodId, sourceId, key ); + if ( group != null && group.getId() > 0 ) + { + return crossTabStore.getAggregatedOrgUnitDataCacheValue( operands, period.getId(), unit.getId(), group.getId(), key ); + } + else + { + return crossTabStore.getAggregatedDataCacheValue( operands, period.getId(), unit.getId(), key ); + } } } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2011-06-06 05:46:14 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2011-12-11 15:48:13 +0000 @@ -43,6 +43,7 @@ final String ID = CrossTabStore.class.getName(); final String CROSSTAB_TABLE_PREFIX = "crosstab_table_"; final String AGGREGATEDDATA_CACHE_PREFIX = "aggregateddata_cache_"; + final String AGGREGATEDORGUNITDATA_CACHE_PREFIX = "aggregatedorgunitdata_cache_"; /** * Creates a crosstab table where the first column is the period identifier, @@ -70,9 +71,28 @@ /** * Drops the aggregated data cache table. + * * @param key the key used in the table name. */ void dropAggregatedDataCache( String key ); + + /** + * Creates a table which functions as a cache for aggregated org unit data + * element values with columns for period identifier, organisation unit + * identifier, organisation unit group identifier followed by one column for + * each DataElementOperand in the given list. + * + * @param operands the list of DataElementOperands. + * @param key the key to use in table name. + */ + void createAggregatedOrgUnitDataCache( List operands, String key ); + + /** + * Drops the aggregated org unit data cache table. + * + * @param key the key used in the table name. + */ + void dropAggregatedOrgUnitDataCache( String key ); /** * Gets all CrossTabDataValues for the given collection of period ids and source ids. @@ -106,5 +126,20 @@ * @param key the key to use in the table name. * @return a map of DataElementOperands and aggregated values. */ - Map getAggregatedDataCacheValue( Collection operands, int periodId, int sourceId, String key ); + Map getAggregatedDataCacheValue( Collection operands, + int periodId, int sourceId, String key ); + + /** + * Gets a map of DataElementOperands and corresponding Double aggregated data + * element value from the cache table. + * + * @param operands the list of DataElementOperand to return map entries for. + * @param periodId the period identifier. + * @param sourceId the organisation unit identifier. + * @param organisationUnitGroupId the organisation unit group identifier. + * @param key the key to use in the table name. + * @return a map of DataElementOperands and aggregated values. + */ + Map getAggregatedOrgUnitDataCacheValue( Collection operands, + int periodId, int sourceId, int organisationUnitGroupId, String key ); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-10-29 14:16:54 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2011-12-11 15:48:13 +0000 @@ -107,6 +107,29 @@ { statementManager.getHolder().executeUpdate( "DROP TABLE IF EXISTS " + AGGREGATEDDATA_CACHE_PREFIX + key ); } + + public void createAggregatedOrgUnitDataCache( List operands, String key ) + { + final StringBuffer sql = new StringBuffer( "CREATE TABLE " + AGGREGATEDORGUNITDATA_CACHE_PREFIX + key + " ( " ); + + sql.append( "periodid INTEGER NOT NULL, " ); + sql.append( "sourceid INTEGER NOT NULL, " ); + sql.append( "organisationunitgroupid INTEGER NOT NULL, " ); + + for ( DataElementOperand operand : operands ) + { + sql.append( operand.getColumnName() ).append( " DOUBLE, " ); + } + + sql.append( "PRIMARY KEY ( periodid, sourceid, organisationunitgroupid ) );" ); + + statementManager.getHolder().executeUpdate( sql.toString() ); + } + + public void dropAggregatedOrgUnitDataCache( String key ) + { + statementManager.getHolder().executeUpdate( "DROP TABLE IF EXISTS " + AGGREGATEDORGUNITDATA_CACHE_PREFIX + key ); + } // ------------------------------------------------------------------------- // CrossTabDataValue @@ -169,28 +192,42 @@ { final StatementHolder holder = statementManager.getHolder(); - final String sql = "SELECT * FROM " + AGGREGATEDDATA_CACHE_PREFIX + key + " AS a WHERE a.periodid = " + periodId + " AND a.sourceid = " + sourceId; - - try - { - final Map valueMap = new HashMap( operands.size() ); - - final ResultSet resultSet = holder.getStatement().executeQuery( sql ); - - if ( resultSet.next() ) - { - for ( DataElementOperand operand : operands ) - { - final Double columnValue = resultSet.getDouble( operand.getColumnName() ); - - if ( columnValue != null ) - { - valueMap.put( operand, columnValue ); - } - } - } - - return valueMap; + // TODO use prepared statement? + + final String sql = "SELECT * FROM " + AGGREGATEDDATA_CACHE_PREFIX + key + + " AS a WHERE a.periodid = " + periodId + " AND a.sourceid = " + sourceId; + + try + { + final ResultSet resultSet = holder.getStatement().executeQuery( sql ); + + return getOperandValueMap( resultSet, operands ); + } + catch ( SQLException ex ) + { + throw new RuntimeException( "Failed to get Map", ex ); + } + finally + { + holder.close(); + } + } + + public Map getAggregatedOrgUnitDataCacheValue( Collection operands, + int periodId, int sourceId, int organisationUnitGroupId, String key ) + { + final StatementHolder holder = statementManager.getHolder(); + + // TODO use prepared statement? + + final String sql = "SELECT * FROM " + AGGREGATEDDATA_CACHE_PREFIX + key + + " AS a WHERE a.periodid = " + periodId + " AND a.sourceid = " + sourceId + " AND a.organisationunitgroupid = " + organisationUnitGroupId; + + try + { + final ResultSet resultSet = holder.getStatement().executeQuery( sql ); + + return getOperandValueMap( resultSet, operands ); } catch ( SQLException ex ) { @@ -234,6 +271,27 @@ return values; } + private Map getOperandValueMap( ResultSet resultSet, Collection operands ) + throws SQLException + { + final Map valueMap = new HashMap( operands.size() ); + + if ( resultSet.next() ) + { + for ( DataElementOperand operand : operands ) + { + final Double columnValue = resultSet.getDouble( operand.getColumnName() ); + + if ( columnValue != null ) + { + valueMap.put( operand, columnValue ); + } + } + } + + return valueMap; + } + private String getCommadelimitedString( Collection operands ) { final StringBuilder builder = new StringBuilder(); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DataMartEngine.java 2011-12-10 18:15:44 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DataMartEngine.java 2011-12-11 01:24:32 +0000 @@ -51,9 +51,11 @@ * @param indicatorIds the indicator identifiers. * @param periodIds the period identifiers. * @param organisationUnitIds the organisation unit identifiers. + * @param organisationUnitGroupIds the organisation unit group identifiers. * @param completeExport indicates whether this is a complete export. * @param processState the state object. */ void export( Collection dataElementIds, Collection indicatorIds, - Collection periodIds, Collection organisationUnitIds, boolean completeExport, ProcessState processState ); + Collection periodIds, Collection organisationUnitIds, Collection organisationUnitGroupIds, + boolean completeExport, ProcessState processState ); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-10 18:15:44 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2011-12-11 16:07:13 +0000 @@ -36,6 +36,7 @@ import java.util.concurrent.Future; import org.hisp.dhis.aggregation.AggregatedDataValueService; +import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueService; import org.hisp.dhis.common.ProcessState; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementCategoryService; @@ -49,11 +50,18 @@ import org.hisp.dhis.indicator.Indicator; import org.hisp.dhis.indicator.IndicatorService; import org.hisp.dhis.jdbc.batchhandler.AggregatedDataValueBatchHandler; +import org.hisp.dhis.jdbc.batchhandler.AggregatedIndicatorValueBatchHandler; +import org.hisp.dhis.jdbc.batchhandler.AggregatedOrgUnitDataValueBatchHandler; +import org.hisp.dhis.jdbc.batchhandler.AggregatedOrgUnitIndicatorValueBatchHandler; +import org.hisp.dhis.options.SystemSettingManager; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; +import org.hisp.dhis.organisationunit.OrganisationUnitGroupService; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.system.filter.AggregatableDataElementFilter; +import org.hisp.dhis.system.filter.OrganisationUnitAboveOrEqualToLevelFilter; import org.hisp.dhis.system.filter.PastAndCurrentPeriodFilter; import org.hisp.dhis.system.util.Clock; import org.hisp.dhis.system.util.ConcurrentUtils; @@ -63,6 +71,8 @@ import org.hisp.dhis.system.util.SystemUtils; import org.springframework.transaction.annotation.Transactional; +import static org.hisp.dhis.options.SystemSettingManager.*; + /** * @author Lars Helge Overland */ @@ -79,6 +89,13 @@ { this.aggregatedDataValueService = aggregatedDataValueService; } + + private AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService; + + public void setAggregatedOrgUnitDataValueService( AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService ) + { + this.aggregatedOrgUnitDataValueService = aggregatedOrgUnitDataValueService; + } private CrossTabService crossTabService; @@ -142,6 +159,20 @@ { this.organisationUnitService = organisationUnitService; } + + private OrganisationUnitGroupService organisationUnitGroupService; + + public void setOrganisationUnitGroupService( OrganisationUnitGroupService organisationUnitGroupService ) + { + this.organisationUnitGroupService = organisationUnitGroupService; + } + + private SystemSettingManager systemSettingManager; + + public void setSystemSettingManager( SystemSettingManager systemSettingManager ) + { + this.systemSettingManager = systemSettingManager; + } // ------------------------------------------------------------------------- // DataMartEngine implementation @@ -149,7 +180,8 @@ @Transactional public void export( Collection dataElementIds, Collection indicatorIds, - Collection periodIds, Collection organisationUnitIds, boolean completeExport, ProcessState state ) + Collection periodIds, Collection organisationUnitIds, Collection organisationUnitGroupIds, + boolean completeExport, ProcessState state ) { final int cpuCores = SystemUtils.getCpuCores(); @@ -162,6 +194,7 @@ Collection indicators = indicatorService.getIndicators( indicatorIds ); Collection periods = periodService.getPeriods( periodIds ); List organisationUnits = new ArrayList( organisationUnitService.getOrganisationUnits( organisationUnitIds ) ); + Collection organisationUnitGroups = organisationUnitGroupService.getOrganisationUnitGroups( organisationUnitGroupIds ); Collection dataElements = dataElementService.getDataElements( dataElementIds ); clock.logTime( "Retrieved objects" ); @@ -244,8 +277,10 @@ clock.logTime( "Populated crosstab table" ); + final boolean isIndicators = indicators != null && indicators.size() > 0; + // --------------------------------------------------------------------- - // Create aggregated data cache + // 1. Create aggregated data cache // --------------------------------------------------------------------- crossTabService.createAggregatedDataCache( indicatorOperands, key ); @@ -253,17 +288,15 @@ clock.logTime( "Created aggregated data cache" ); // --------------------------------------------------------------------- - // Drop potential indexes + // 2. Drop potential indexes // --------------------------------------------------------------------- - final boolean isIndicators = indicators != null && indicators.size() > 0; - aggregatedDataValueService.dropIndex( true, isIndicators ); clock.logTime( "Dropped potential indexes" ); // --------------------------------------------------------------------- - // Delete existing aggregated datavalues + // 3. Delete existing aggregated datavalues // --------------------------------------------------------------------- if ( completeExport ) @@ -278,7 +311,7 @@ clock.logTime( "Deleted existing aggregated datavalues" ); // --------------------------------------------------------------------- - // Export data element values + // 4. Export data element values // --------------------------------------------------------------------- state.setMessage( "exporting_data_for_data_elements" ); @@ -301,15 +334,7 @@ } // --------------------------------------------------------------------- - // Drop crosstab table - // --------------------------------------------------------------------- - - crossTabService.dropCrossTabTable( key ); - - clock.logTime( "Dropped crosstab table" ); - - // --------------------------------------------------------------------- - // Delete existing aggregated indicatorvalues + // 5. Delete existing aggregated indicatorvalues // --------------------------------------------------------------------- if ( completeExport ) @@ -324,7 +349,7 @@ clock.logTime( "Deleted existing aggregated indicatorvalues" ); // --------------------------------------------------------------------- - // Export indicator values + // 6. Export indicator values // --------------------------------------------------------------------- state.setMessage( "exporting_data_for_indicators" ); @@ -335,7 +360,8 @@ for ( List organisationUnitPage : organisationUnitPages ) { - futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, indicatorOperands, key ) ); + futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, + null, indicatorOperands, AggregatedIndicatorValueBatchHandler.class, key ) ); } ConcurrentUtils.waitForCompletion( futures ); @@ -344,7 +370,7 @@ } // --------------------------------------------------------------------- - // Drop aggregated data cache + // 7. Drop aggregated data cache // --------------------------------------------------------------------- crossTabService.dropAggregatedDataCache( key ); @@ -352,7 +378,7 @@ clock.logTime( "Dropped aggregated data cache" ); // --------------------------------------------------------------------- - // Create potential indexes + // 8. Create potential indexes // --------------------------------------------------------------------- if ( completeExport ) @@ -362,6 +388,134 @@ clock.logTime( "Created indexes" ); } + final boolean isGroups = organisationUnitGroupIds != null && organisationUnitGroupIds.size() > 0; + + final int groupLevel = (Integer) systemSettingManager.getSystemSetting( KEY_ORGUNITGROUPSET_AGG_LEVEL, DEFAULT_ORGUNITGROUPSET_AGG_LEVEL ); + + if ( isGroups && groupLevel > 0 ) + { + // ----------------------------------------------------------------- + // 1. Create aggregated data cache + // ----------------------------------------------------------------- + + crossTabService.createAggregatedOrgUnitDataCache( indicatorOperands, key ); + + clock.logTime( "Created aggregated org unit data cache" ); + + // ----------------------------------------------------------------- + // 2. Drop potential indexes + // ----------------------------------------------------------------- + + aggregatedOrgUnitDataValueService.dropIndex( true, isIndicators ); + + clock.logTime( "Dropped potential org unit indexes" ); + + // --------------------------------------------------------------------- + // 3. Delete existing aggregated datavalues + // --------------------------------------------------------------------- + + if ( completeExport ) + { + aggregatedOrgUnitDataValueService.deleteAggregatedDataValues( periodIds ); + } + else + { + aggregatedOrgUnitDataValueService.deleteAggregatedDataValues( dataElementIds, periodIds, organisationUnitIds ); + } + + clock.logTime( "Deleted existing aggregated org unit datavalues" ); + + // --------------------------------------------------------------------- + // 4. Export data element values + // --------------------------------------------------------------------- + + state.setMessage( "exporting_data_for_data_elements" ); + + Collection groupOrganisationUnits = new HashSet( organisationUnits ); + + FilterUtils.filter( groupOrganisationUnits, new OrganisationUnitAboveOrEqualToLevelFilter( groupLevel ) ); + + organisationUnitPages = new PaginatedList( groupOrganisationUnits ).setNumberOfPages( cpuCores ).getPages(); + + if ( allOperands.size() > 0 ) + { + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( dataElementDataMart.exportDataValues( allOperands, periods, organisationUnitPage, + organisationUnitGroups, new DataElementOperandList( indicatorOperands ), AggregatedOrgUnitDataValueBatchHandler.class, key ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + + clock.logTime( "Exported values for data element operands (" + allOperands.size() + "), number of pages: " + organisationUnitPages.size() ); + } + + // --------------------------------------------------------------------- + // 5. Delete existing aggregated indicatorvalues + // --------------------------------------------------------------------- + + if ( completeExport ) + { + aggregatedOrgUnitDataValueService.deleteAggregatedIndicatorValues( periodIds ); + } + else + { + aggregatedOrgUnitDataValueService.deleteAggregatedIndicatorValues( indicatorIds, periodIds, organisationUnitIds ); + } + + clock.logTime( "Deleted existing aggregated org unit indicatorvalues" ); + + // --------------------------------------------------------------------- + // 6. Export indicator values + // --------------------------------------------------------------------- + + state.setMessage( "exporting_data_for_indicators" ); + + if ( isIndicators ) + { + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( indicatorDataMart.exportIndicatorValues( indicators, periods, organisationUnitPage, + organisationUnitGroups, indicatorOperands, AggregatedOrgUnitIndicatorValueBatchHandler.class, key ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + + clock.logTime( "Exported values for indicators (" + indicators.size() + "), number of pages: " + organisationUnitPages.size() ); + } + + // --------------------------------------------------------------------- + // 7. Drop aggregated data cache + // --------------------------------------------------------------------- + + crossTabService.dropAggregatedOrgUnitDataCache( key ); + + clock.logTime( "Dropped aggregated org unit data cache" ); + + // --------------------------------------------------------------------- + // 8. Create potential indexes + // --------------------------------------------------------------------- + + if ( completeExport ) + { + aggregatedOrgUnitDataValueService.createIndex( true, isIndicators ); + + clock.logTime( "Created org unit indexes" ); + } + } + + // --------------------------------------------------------------------- + // Drop crosstab table + // --------------------------------------------------------------------- + + crossTabService.dropCrossTabTable( key ); + + clock.logTime( "Dropped crosstab table" ); + clock.logTime( "Data mart export process completed" ); } } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/impl/DefaultDataMartService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/impl/DefaultDataMartService.java 2011-11-03 14:35:44 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/impl/DefaultDataMartService.java 2011-12-11 01:24:32 +0000 @@ -97,6 +97,7 @@ getIdentifiers( Indicator.class, dataMartExport.getIndicators() ), getIdentifiers( Period.class, allPeriods ), getIdentifiers( OrganisationUnit.class, dataMartExport.getOrganisationUnits() ), + null, false, new OutputHolderState() ); } @@ -104,7 +105,7 @@ public void export( Collection dataElementIds, Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ) { - dataMartEngine.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, false, new OutputHolderState() ); + dataMartEngine.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, null, false, new OutputHolderState() ); } public void export( Collection dataElementIds, Collection indicatorIds, @@ -121,7 +122,7 @@ periodIds.addAll( getIdentifiers( Period.class, periodService.reloadPeriods( relatives.getRelativePeriods() ) ) ); } - dataMartEngine.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, completeExport, new OutputHolderState() ); + dataMartEngine.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, null, completeExport, new OutputHolderState() ); } // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java 2011-11-25 10:09:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/DefaultIndicatorDataMart.java 2011-12-11 16:07:13 +0000 @@ -47,11 +47,12 @@ import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.datamart.aggregation.cache.AggregationCache; import org.hisp.dhis.datamart.crosstab.CrossTabService; +import org.hisp.dhis.datamart.engine.DataMartEngine; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.indicator.Indicator; -import org.hisp.dhis.jdbc.batchhandler.AggregatedIndicatorValueBatchHandler; import org.hisp.dhis.options.SystemSettingManager; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.system.util.DateUtils; @@ -128,18 +129,21 @@ // ------------------------------------------------------------------------- @Async - public Future exportIndicatorValues( final Collection indicators, final Collection periods, - final Collection organisationUnits, final Collection operands, String key ) + public Future exportIndicatorValues( Collection indicators, Collection periods, + Collection organisationUnits, Collection organisationUnitGroups, + Collection operands, Class> clazz, String key ) { statementManager.initialise(); // Running in separate thread - final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( AggregatedIndicatorValueBatchHandler.class ).init(); + final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( clazz ).init(); final boolean omitZeroNumerator = (Boolean) systemSettingManager.getSystemSetting( KEY_OMIT_INDICATORS_ZERO_NUMERATOR_DATAMART, false ); final AggregatedIndicatorValue indicatorValue = new AggregatedIndicatorValue(); final Map constantMap = constantService.getConstantMap(); + + organisationUnitGroups = ( organisationUnitGroups != null ) ? organisationUnitGroups : DataMartEngine.DUMMY_ORG_UNIT_GROUPS; for ( final Period period : periods ) { @@ -147,43 +151,47 @@ final PeriodType periodType = period.getPeriodType(); - for ( final OrganisationUnit unit : organisationUnits ) + for ( OrganisationUnitGroup group : organisationUnitGroups ) { - final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() ); - - final Map valueMap = crossTabService.getAggregatedDataCacheValue( operands, period.getId(), unit.getId(), key ); - - if ( valueMap.size() > 0 ) - { - for ( final Indicator indicator : indicators ) - { - final double denominatorValue = calculateExpression( expressionService.generateExpression( indicator.getExplodedDenominator(), valueMap, constantMap, days ) ); - - if ( !isEqual( denominatorValue, 0d ) ) + for ( final OrganisationUnit unit : organisationUnits ) + { + final int level = aggregationCache.getLevelOfOrganisationUnit( unit.getId() ); + + final Map valueMap = crossTabService.getAggregatedDataCacheValue( operands, period, unit, group, key ); + + if ( valueMap.size() > 0 ) + { + for ( final Indicator indicator : indicators ) { - final double numeratorValue = calculateExpression( expressionService.generateExpression( indicator.getExplodedNumerator(), valueMap, constantMap, days ) ); - - if ( !( omitZeroNumerator && isEqual( numeratorValue, 0d ) ) ) + final double denominatorValue = calculateExpression( expressionService.generateExpression( indicator.getExplodedDenominator(), valueMap, constantMap, days ) ); + + if ( !isEqual( denominatorValue, 0d ) ) { - final double annualizationFactor = DateUtils.getAnnualizationFactor( indicator, period.getStartDate(), period.getEndDate() ); - final double factor = indicator.getIndicatorType().getFactor(); - final double aggregatedValue = ( numeratorValue / denominatorValue ) * factor * annualizationFactor; - final double annualizedFactor = factor * annualizationFactor; - - indicatorValue.clear(); - - indicatorValue.setIndicatorId( indicator.getId() ); - indicatorValue.setPeriodId( period.getId() ); - indicatorValue.setPeriodTypeId( periodType.getId() ); - indicatorValue.setOrganisationUnitId( unit.getId() ); - indicatorValue.setLevel( level ); - indicatorValue.setAnnualized( getAnnualizationString( indicator.isAnnualized() ) ); - indicatorValue.setFactor( annualizedFactor); - indicatorValue.setValue( getRounded( aggregatedValue, DECIMALS ) ); - indicatorValue.setNumeratorValue( getRounded( numeratorValue, DECIMALS ) ); - indicatorValue.setDenominatorValue( getRounded( denominatorValue, DECIMALS ) ); - - batchHandler.addObject( indicatorValue ); + final double numeratorValue = calculateExpression( expressionService.generateExpression( indicator.getExplodedNumerator(), valueMap, constantMap, days ) ); + + if ( !( omitZeroNumerator && isEqual( numeratorValue, 0d ) ) ) + { + final double annualizationFactor = DateUtils.getAnnualizationFactor( indicator, period.getStartDate(), period.getEndDate() ); + final double factor = indicator.getIndicatorType().getFactor(); + final double aggregatedValue = ( numeratorValue / denominatorValue ) * factor * annualizationFactor; + final double annualizedFactor = factor * annualizationFactor; + + indicatorValue.clear(); + + indicatorValue.setIndicatorId( indicator.getId() ); + indicatorValue.setPeriodId( period.getId() ); + indicatorValue.setPeriodTypeId( periodType.getId() ); + indicatorValue.setOrganisationUnitId( unit.getId() ); + indicatorValue.setOrganisationUnitGroupId( group != null ? group.getId() : 0 ); + indicatorValue.setLevel( level ); + indicatorValue.setAnnualized( getAnnualizationString( indicator.isAnnualized() ) ); + indicatorValue.setFactor( annualizedFactor); + indicatorValue.setValue( getRounded( aggregatedValue, DECIMALS ) ); + indicatorValue.setNumeratorValue( getRounded( numeratorValue, DECIMALS ) ); + indicatorValue.setDenominatorValue( getRounded( denominatorValue, DECIMALS ) ); + + batchHandler.addObject( indicatorValue ); + } } } } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/IndicatorDataMart.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/IndicatorDataMart.java 2011-07-01 11:20:25 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/indicator/IndicatorDataMart.java 2011-12-11 16:07:13 +0000 @@ -30,9 +30,12 @@ import java.util.Collection; import java.util.concurrent.Future; +import org.amplecode.quick.BatchHandler; +import org.hisp.dhis.aggregation.AggregatedIndicatorValue; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.indicator.Indicator; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitGroup; import org.hisp.dhis.period.Period; /** @@ -41,5 +44,6 @@ public interface IndicatorDataMart { Future exportIndicatorValues( Collection indicators, Collection periods, - Collection organisationUnits, Collection operands, String key ); + Collection organisationUnits, Collection organisationUnitGroups, + Collection operands, Class> clazz, String key ); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-11-21 12:44:20 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-12-11 16:07:13 +0000 @@ -33,6 +33,8 @@ + + === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-10-29 14:16:54 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/test/java/org/hisp/dhis/datamart/crosstab/CrossTabServiceTest.java 2011-12-11 16:07:13 +0000 @@ -57,6 +57,7 @@ import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.MonthlyPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.period.WeeklyPeriodType; @@ -220,6 +221,12 @@ public void testPopulateAggregatedDataCache() { String key = RandomStringUtils.randomAlphanumeric( 8 ); + + Period period = new MonthlyPeriodType().createPeriod(); + period.setId( 1 ); + + OrganisationUnit unit = createOrganisationUnit( 'A' ); + unit.setId( 1 ); crossTabService.createAggregatedDataCache( operands, key ); @@ -239,7 +246,7 @@ batchHandler.flush(); - Map valueMap = crossTabService.getAggregatedDataCacheValue( operands, 1, 1, key ); + Map valueMap = crossTabService.getAggregatedDataCacheValue( operands, period, unit, null, key ); for ( DataElementOperand operand : valueMap.keySet() ) { === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitConverter.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitConverter.java 2011-11-22 12:46:08 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitConverter.java 2011-12-11 01:24:32 +0000 @@ -110,7 +110,7 @@ { for ( OrganisationUnit unit : units ) { - int level = organisationUnitService.getLevelOfOrganisationUnit( unit ); + int level = unit.getOrganisationUnitLevel(); writer.openElement( ELEMENT_NAME ); === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitHierarchyConverter.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitHierarchyConverter.java 2011-02-20 18:57:52 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/OrganisationUnitHierarchyConverter.java 2011-12-11 01:24:32 +0000 @@ -113,7 +113,7 @@ { if ( unit.getParent() != null ) { - int level = organisationUnitService.getLevelOfOrganisationUnit( unit ); + int level = unit.getOrganisationUnitLevel(); writer.openElement( ELEMENT_NAME ); === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/synchronous/ExportPivotViewService.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/synchronous/ExportPivotViewService.java 2011-10-13 18:35:12 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/synchronous/ExportPivotViewService.java 2011-12-11 01:24:32 +0000 @@ -118,7 +118,7 @@ return; } - rootOrgUnit.setLevel( organisationUnitService.getLevelOfOrganisationUnit( rootOrgUnit ) ); + rootOrgUnit.setLevel( rootOrgUnit.getOrganisationUnitLevel() ); OrganisationUnitLevel orgUnitLevel = organisationUnitService.getOrganisationUnitLevelByLevel( level ); @@ -157,7 +157,7 @@ return 0; } - rootOrgUnit.setLevel( organisationUnitService.getLevelOfOrganisationUnit( rootOrgUnit ) ); + rootOrgUnit.setLevel( rootOrgUnit.getOrganisationUnitLevel() ); OrganisationUnitLevel orgUnitLevel = organisationUnitService.getOrganisationUnitLevelByLevel( level ); === modified file 'dhis-2/dhis-services/dhis-service-mapping/src/main/java/org/hisp/dhis/mapping/DefaultMappingService.java' --- dhis-2/dhis-services/dhis-service-mapping/src/main/java/org/hisp/dhis/mapping/DefaultMappingService.java 2011-12-06 17:41:41 +0000 +++ dhis-2/dhis-services/dhis-service-mapping/src/main/java/org/hisp/dhis/mapping/DefaultMappingService.java 2011-12-11 13:47:04 +0000 @@ -508,8 +508,8 @@ { if ( mapView != null ) { - mapView.getParentOrganisationUnit().setLevel( - organisationUnitService.getLevelOfOrganisationUnit( mapView.getParentOrganisationUnit() ) ); + mapView.getParentOrganisationUnit().setLevel( organisationUnitService.getLevelOfOrganisationUnit( + mapView.getParentOrganisationUnit().getId() ) ); } } @@ -528,8 +528,8 @@ { for ( MapView mapView : mapViews ) { - mapView.getParentOrganisationUnit().setLevel( - organisationUnitService.getLevelOfOrganisationUnit( mapView.getParentOrganisationUnit() ) ); + mapView.getParentOrganisationUnit().setLevel( organisationUnitService.getLevelOfOrganisationUnit( + mapView.getParentOrganisationUnit().getId() ) ); } } @@ -544,8 +544,8 @@ for ( MapView mapView : mapViews ) { - mapView.getParentOrganisationUnit().setLevel( - organisationUnitService.getLevelOfOrganisationUnit( mapView.getParentOrganisationUnit() ) ); + mapView.getParentOrganisationUnit().setLevel( organisationUnitService.getLevelOfOrganisationUnit( + mapView.getParentOrganisationUnit().getId() ) ); } return mapViews; === added file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/filter/OrganisationUnitAboveOrEqualToLevelFilter.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/filter/OrganisationUnitAboveOrEqualToLevelFilter.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/filter/OrganisationUnitAboveOrEqualToLevelFilter.java 2011-12-11 16:07:13 +0000 @@ -0,0 +1,55 @@ +package org.hisp.dhis.system.filter; + +/* + * Copyright (c) 2004-2010, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.system.util.Filter; + +/** + * Retains organisation units which are above or at the same level as the level + * given in the constructor. This filter must be invoked inside a transactional + * context in order to work. + * + * @author Lars Helge Overland + */ +public class OrganisationUnitAboveOrEqualToLevelFilter + implements Filter +{ + private int level; + + public OrganisationUnitAboveOrEqualToLevelFilter( int level ) + { + this.level = level; + } + + @Override + public boolean retain( OrganisationUnit object ) + { + return object != null && object.getOrganisationUnitLevel() >= level; + } +}