=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueService.java 2012-07-25 16:22:03 +0000 @@ -178,13 +178,6 @@ Collection organisationUnitIds ); /** - * Deletes AggregatedDataValues registered for the given parameters. - * - * @param periodIds a collection of Period identifiers. - */ - void deleteAggregatedDataValues( Collection periodIds ); - - /** * Deletes all AggregatedDataValues. * * @return the number of deleted AggregatedDataValues. @@ -209,24 +202,6 @@ */ public int countDataValuesAtLevel( OrganisationUnit orgunit, OrganisationUnitLevel level, Collection periods ); - /** - * Creates indexes on the aggregateddatavalue and aggregatedindicatorvalue - * tables. - * - * @param dataElement indicates whether to create an index on aggregateddatavalue. - * @param indicator indicates whether to create an index on aggregatedindicatorvalue. - */ - void createIndex( boolean dataElement, boolean indicator ); - - /** - * Drops the indexes on the aggregateddatavalue and aggregatedindicatorvalue - * tables. - * - * @param dataElement indicates whether to drop the index on aggregateddatavalue. - * @param indicator indicates whether to drop the index on aggregatedindicatorvalue. - */ - void dropIndex( boolean dataElement, boolean indicator ); - // ---------------------------------------------------------------------- // AggregatedDataMapValue // ---------------------------------------------------------------------- @@ -294,13 +269,6 @@ Collection organisationUnitIds ); /** - * Deletes AggregatedIndicatorValue registered for the given parameters. - * - * @param periodIds a collection of Period identifiers. - */ - void deleteAggregatedIndicatorValues( Collection periodIds ); - - /** * Deletes all AggregatedIndicatorValue. * * @return the number of deleted AggregatedIndicatorValues. === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedDataValueStore.java 2012-07-25 16:22:03 +0000 @@ -155,13 +155,6 @@ Collection organisationUnitIds ); /** - * Deletes AggregatedDataValues registered for the given parameters. - * - * @param periodIds a collection of Period identifiers. - */ - void deleteAggregatedDataValues( Collection periodIds ); - - /** * Deletes all AggregatedDataValues. * * @return the number of deleted AggregatedDataValues. @@ -186,24 +179,6 @@ */ public int countDataValuesAtLevel( OrganisationUnit orgunit, OrganisationUnitLevel level, Collection periods ); - /** - * Creates indexes on the aggregateddatavalue and aggregatedindicatorvalue - * tables. - * - * @param dataElement indicates whether to create an index on aggregateddatavalue. - * @param indicator indicates whether to create an index on aggregatedindicatorvalue. - */ - void createIndex( boolean dataElement, boolean indicator ); - - /** - * Drops the indexes on the aggregateddatavalue and aggregatedindicatorvalue - * tables. - * - * @param dataElement indicates whether to drop the index on aggregateddatavalue. - * @param indicator indicates whether to drop the index on aggregatedindicatorvalue. - */ - void dropIndex( boolean dataElement, boolean indicator ); - // ---------------------------------------------------------------------- // AggregatedDataMapValue // ---------------------------------------------------------------------- @@ -271,13 +246,6 @@ Collection organisationUnitIds ); /** - * Deletes AggregatedIndicatorValue registered for the given parameters. - * - * @param periodIds a collection of Period identifiers. - */ - void deleteAggregatedIndicatorValues( Collection periodIds ); - - /** * Deletes all AggregatedIndicatorValue. * * @return the number of deleted AggregatedIndicatorValues. === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java 2012-02-29 12:26:39 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueService.java 2012-07-25 16:22:03 +0000 @@ -52,14 +52,8 @@ void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ); - void deleteAggregatedDataValues( Collection periodIds ); - void deleteAggregatedDataValues(); - void createIndex( boolean dataElement, boolean indicator ); - - void dropIndex( boolean dataElement, boolean indicator ); - // ------------------------------------------------------------------------- // AggregatedIndicatorValue // ------------------------------------------------------------------------- @@ -71,7 +65,5 @@ void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ); - void deleteAggregatedIndicatorValues( Collection periodIds ); - void deleteAggregatedIndicatorValues(); } === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java 2012-02-29 12:26:39 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/aggregation/AggregatedOrgUnitDataValueStore.java 2012-07-25 16:22:03 +0000 @@ -42,14 +42,8 @@ void deleteAggregatedDataValues( Collection dataElementIds, Collection periodIds, Collection organisationUnitIds ); - void deleteAggregatedDataValues( Collection periodIds ); - void deleteAggregatedDataValues(); - void createIndex( boolean dataElement, boolean indicator ); - - void dropIndex( boolean dataElement, boolean indicator ); - // ------------------------------------------------------------------------- // AggregatedIndicatorValue // ------------------------------------------------------------------------- @@ -61,7 +55,5 @@ void deleteAggregatedIndicatorValues( Collection indicatorIds, Collection periodIds, Collection organisationUnitIds ); - void deleteAggregatedIndicatorValues( Collection periodIds ); - void deleteAggregatedIndicatorValues(); } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedDataValueService.java 2012-07-25 16:22:03 +0000 @@ -120,11 +120,6 @@ { aggregatedDataValueStore.deleteAggregatedDataValues( dataElementIds, periodIds, organisationUnitIds ); } - - public void deleteAggregatedDataValues( Collection periodIds ) - { - aggregatedDataValueStore.deleteAggregatedDataValues( periodIds ); - } public void deleteAggregatedDataValues() { @@ -140,17 +135,7 @@ { return aggregatedDataValueStore.countDataValuesAtLevel( orgunit, level, periods ); } - - public void createIndex( boolean dataElement, boolean indicator ) - { - aggregatedDataValueStore.createIndex( dataElement, indicator ); - } - - public void dropIndex( boolean dataElement, boolean indicator ) - { - aggregatedDataValueStore.dropIndex( dataElement, indicator ); - } - + // ------------------------------------------------------------------------- // AggregatedDataMapValue // ------------------------------------------------------------------------- @@ -195,11 +180,6 @@ { aggregatedDataValueStore.deleteAggregatedIndicatorValues( indicatorIds, periodIds, organisationUnitIds ); } - - public void deleteAggregatedIndicatorValues( Collection periodIds ) - { - aggregatedDataValueStore.deleteAggregatedIndicatorValues( periodIds ); - } public void deleteAggregatedIndicatorValues() { === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java 2012-02-29 12:26:39 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/DefaultAggregatedOrgUnitDataValueService.java 2012-07-25 16:22:03 +0000 @@ -74,26 +74,11 @@ aggregatedDataValueStore.deleteAggregatedDataValues( dataElementIds, periodIds, organisationUnitIds ); } - public void deleteAggregatedDataValues( Collection periodIds ) - { - aggregatedDataValueStore.deleteAggregatedDataValues( periodIds ); - } - public void deleteAggregatedDataValues() { aggregatedDataValueStore.deleteAggregatedDataValues(); } - public void createIndex( boolean dataElement, boolean indicator ) - { - aggregatedDataValueStore.createIndex( dataElement, indicator ); - } - - public void dropIndex( boolean dataElement, boolean indicator ) - { - aggregatedDataValueStore.dropIndex( dataElement, indicator ); - } - // ------------------------------------------------------------------------- // AggregatedIndicatorValue // ------------------------------------------------------------------------- @@ -113,12 +98,7 @@ { aggregatedDataValueStore.deleteAggregatedIndicatorValues( indicatorIds, periodIds, organisationUnitIds ); } - - public void deleteAggregatedIndicatorValues( Collection periodIds ) - { - aggregatedDataValueStore.deleteAggregatedIndicatorValues( periodIds ); - } - + public void deleteAggregatedIndicatorValues() { aggregatedDataValueStore.deleteAggregatedIndicatorValues(); === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedDataValueStore.java 2012-07-25 16:22:03 +0000 @@ -39,8 +39,6 @@ import org.amplecode.quick.StatementManager; import org.amplecode.quick.mapper.ObjectMapper; import org.amplecode.quick.mapper.RowMapper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.hisp.dhis.aggregation.AggregatedDataValue; import org.hisp.dhis.aggregation.AggregatedDataValueStore; import org.hisp.dhis.aggregation.AggregatedIndicatorValue; @@ -72,8 +70,6 @@ { private int FETCH_SIZE = 1000; // Number of rows to fetch from db for large resultset - private static final Log log = LogFactory.getLog( JdbcAggregatedDataValueStore.class ); - // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- @@ -359,15 +355,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void deleteAggregatedDataValues( Collection periodIds ) - { - final String sql = - "DELETE FROM aggregateddatavalue " + - "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; - - statementManager.getHolder().executeUpdate( sql ); - } - public void deleteAggregatedDataValues() { final String sql = "DELETE FROM aggregateddatavalue"; @@ -375,64 +362,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void createIndex( boolean dataElement, boolean indicator ) - { - if ( dataElement ) - { - try - { - final String sql = "CREATE INDEX aggregateddatavalue_index ON aggregateddatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid)"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index already exists" ); - } - } - - if ( indicator ) - { - try - { - final String sql = "CREATE INDEX aggregatedindicatorvalue_index ON aggregatedindicatorvalue (indicatorid, periodid, organisationunitid)"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index already exists" ); - } - } - } - - public void dropIndex( boolean dataElement, boolean indicator ) - { - if ( dataElement ) - { - try - { - final String sql = "DROP INDEX aggregateddatavalue_index"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index does not exist" ); - } - } - - if ( indicator ) - { - try - { - final String sql = "DROP INDEX aggregatedindicatorvalue_index"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index does not exist" ); - } - } - } - // ------------------------------------------------------------------------- // AggregatedDataMapValue // ------------------------------------------------------------------------- @@ -562,15 +491,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void deleteAggregatedIndicatorValues( Collection periodIds ) - { - final String sql = - "DELETE FROM aggregatedindicatorvalue " + - "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; - - statementManager.getHolder().executeUpdate( sql ); - } - public void deleteAggregatedIndicatorValues() { final String sql = "DELETE FROM aggregatedindicatorvalue"; === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2012-03-08 14:43:53 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/aggregation/jdbc/JdbcAggregatedOrgUnitDataValueStore.java 2012-07-25 16:22:03 +0000 @@ -36,8 +36,6 @@ import org.amplecode.quick.StatementHolder; import org.amplecode.quick.StatementManager; import org.amplecode.quick.mapper.ObjectMapper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.hisp.dhis.aggregation.AggregatedDataValue; import org.hisp.dhis.aggregation.AggregatedIndicatorValue; import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueStore; @@ -47,8 +45,6 @@ public class JdbcAggregatedOrgUnitDataValueStore implements AggregatedOrgUnitDataValueStore { - private static final Log log = LogFactory.getLog( JdbcAggregatedOrgUnitDataValueStore.class ); - // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- @@ -121,15 +117,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void deleteAggregatedDataValues( Collection periodIds ) - { - final String sql = - "DELETE FROM aggregatedorgunitdatavalue " + - "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; - - statementManager.getHolder().executeUpdate( sql ); - } - public void deleteAggregatedDataValues() { final String sql = "DELETE FROM aggregatedorgunitdatavalue"; @@ -137,64 +124,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void createIndex( boolean dataElement, boolean indicator ) - { - if ( dataElement ) - { - try - { - final String sql = "CREATE INDEX aggregatedorgunitdatavalue_index ON aggregatedorgunitdatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid, organisationunitgroupid)"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index already exists" ); - } - } - - if ( indicator ) - { - try - { - final String sql = "CREATE INDEX aggregatedorgunitindicatorvalue_index ON aggregatedorgunitindicatorvalue (indicatorid, periodid, organisationunitid, organisationunitgroupid)"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index already exists" ); - } - } - } - - public void dropIndex( boolean dataElement, boolean indicator ) - { - if ( dataElement ) - { - try - { - final String sql = "DROP INDEX aggregatedorgunitdatavalue_index"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index does not exist" ); - } - } - - if ( indicator ) - { - try - { - final String sql = "DROP INDEX aggregatedorgunitindicatorvalue_index"; - statementManager.getHolder().executeUpdate( sql, true ); - } - catch ( Exception ex ) - { - log.debug( "Index does not exist" ); - } - } - } - // ------------------------------------------------------------------------- // AggregatedIndicatorValue // ------------------------------------------------------------------------- @@ -255,15 +184,6 @@ statementManager.getHolder().executeUpdate( sql ); } - public void deleteAggregatedIndicatorValues( Collection periodIds ) - { - final String sql = - "DELETE FROM aggregatedorgunitindicatorvalue " + - "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; - - statementManager.getHolder().executeUpdate( sql ); - } - public void deleteAggregatedIndicatorValues() { final String sql = "DELETE FROM aggregatedorgunitindicatorvalue "; === added file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/DataMartManager.java 2012-07-25 16:22:03 +0000 @@ -0,0 +1,61 @@ +package org.hisp.dhis.datamart; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import org.hisp.dhis.dataelement.DataElementOperand; + +public interface DataMartManager +{ + /** + * Filters and returns the DataElementOperands with data from the given + * collection of DataElementOperands. + * + * @param operands the DataElementOperands. + * @return the DataElementOperands with data. + */ + Set getOperandsWithData( Set operands ); + + Map getDataValueMap( int periodId, int sourceId ); + + /** + * Creates indexes on the aggregateddatavalue and aggregatedindicatorvalue + * tables. + * + * @param dataElement indicates whether to create an index on aggregateddatavalue. + * @param indicator indicates whether to create an index on aggregatedindicatorvalue. + */ + void createAggregatedValueIndex( boolean dataElement, boolean indicator ); + + /** + * Drops the indexes on the aggregateddatavalue and aggregatedindicatorvalue + * tables. + * + * @param dataElement indicates whether to drop the index on aggregateddatavalue. + * @param indicator indicates whether to drop the index on aggregatedindicatorvalue. + */ + void dropAggregatedValueIndex( boolean dataElement, boolean indicator ); + + /** + * Deletes AggregatedDataValues registered for the given parameters. + * + * @param periodIds a collection of Period identifiers. + */ + void deleteAggregatedDataValues( Collection periodIds ); + + /** + * Deletes AggregatedIndicatorValue registered for the given parameters. + * + * @param periodIds a collection of Period identifiers. + */ + void deleteAggregatedIndicatorValues( Collection periodIds ); + + void createAggregatedOrgUnitValueIndex( boolean dataElement, boolean indicator ); + + void dropAggregatedOrgUnitValueIndex( boolean dataElement, boolean indicator ); + + void deleteAggregatedOrgUnitIndicatorValues( Collection periodIds ); + + void deleteAggregatedOrgUnitDataValues( Collection periodIds ); +} === added file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/JdbcDataMartManager.java 2012-07-25 16:22:03 +0000 @@ -0,0 +1,289 @@ +package org.hisp.dhis.datamart; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.amplecode.quick.StatementHolder; +import org.amplecode.quick.StatementManager; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.dataelement.DataElementOperand; + +/** + * @author Lars Helge Overland + */ +public class JdbcDataMartManager + implements DataMartManager +{ + private static final Log log = LogFactory.getLog( JdbcDataMartManager.class ); + + // ------------------------------------------------------------------------- + // Dependencies + // ------------------------------------------------------------------------- + + private StatementManager statementManager; + + public void setStatementManager( StatementManager statementManager ) + { + this.statementManager = statementManager; + } + + // ------------------------------------------------------------------------- + // DataMartManager implementation + // ------------------------------------------------------------------------- + + @Override + public Set getOperandsWithData( Set operands ) + { + final Set operandsWithData = new HashSet(); + + final StatementHolder holder = statementManager.getHolder(); + + for ( DataElementOperand operand : operands ) + { + final String sql = + "SELECT COUNT(*) FROM datavalue " + + "WHERE dataelementid=" + operand.getDataElementId() + " " + + "AND categoryoptioncomboid=" + operand.getOptionComboId(); + + Integer count = holder.queryForInteger( sql ); + + if ( count != null && count > 0 ) + { + operandsWithData.add( operand ); + } + } + + return operandsWithData; + } + + @Override + public Map getDataValueMap( int periodId, int sourceId ) + { + final StatementHolder holder = statementManager.getHolder(); + + try + { + final String sql = + "SELECT dataelementid, categoryoptioncomboid, value " + + "FROM datavalue " + + "WHERE periodid = " + periodId + " " + + "AND sourceid = " + sourceId; + + final ResultSet resultSet = holder.getStatement().executeQuery( sql ); + + final Map map = new HashMap(); + + while ( resultSet.next() ) + { + final DataElementOperand operand = new DataElementOperand( resultSet.getInt( 1 ), resultSet.getInt( 2 ) ); + + map.put( operand, resultSet.getString( 3 ) ); + } + + return map; + } + catch ( SQLException ex ) + { + throw new RuntimeException( "Failed to get DataValues", ex ); + } + finally + { + holder.close(); + } + } + + @Override + public void createAggregatedValueIndex( boolean dataElement, boolean indicator ) + { + if ( dataElement ) + { + try + { + final String sql = "CREATE INDEX aggregateddatavalue_index ON aggregateddatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid)"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index already exists" ); + } + } + + if ( indicator ) + { + try + { + final String sql = "CREATE INDEX aggregatedindicatorvalue_index ON aggregatedindicatorvalue (indicatorid, periodid, organisationunitid)"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index already exists" ); + } + } + } + + @Override + public void dropAggregatedValueIndex( boolean dataElement, boolean indicator ) + { + if ( dataElement ) + { + try + { + final String sql = "DROP INDEX aggregateddatavalue_index"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index does not exist" ); + } + } + + if ( indicator ) + { + try + { + final String sql = "DROP INDEX aggregatedindicatorvalue_index"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index does not exist" ); + } + } + } + + @Override + public void deleteAggregatedDataValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregateddatavalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + statementManager.getHolder().executeUpdate( sql ); + } + + @Override + public void deleteAggregatedIndicatorValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregatedindicatorvalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + statementManager.getHolder().executeUpdate( sql ); + } + + @Override + public void createAggregatedOrgUnitValueIndex( boolean dataElement, boolean indicator ) + { + if ( dataElement ) + { + try + { + final String sql = "CREATE INDEX aggregatedorgunitdatavalue_index ON aggregatedorgunitdatavalue (dataelementid, categoryoptioncomboid, periodid, organisationunitid, organisationunitgroupid)"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index already exists" ); + } + } + + if ( indicator ) + { + try + { + final String sql = "CREATE INDEX aggregatedorgunitindicatorvalue_index ON aggregatedorgunitindicatorvalue (indicatorid, periodid, organisationunitid, organisationunitgroupid)"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index already exists" ); + } + } + } + + @Override + public void dropAggregatedOrgUnitValueIndex( boolean dataElement, boolean indicator ) + { + if ( dataElement ) + { + try + { + final String sql = "DROP INDEX aggregatedorgunitdatavalue_index"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index does not exist" ); + } + } + + if ( indicator ) + { + try + { + final String sql = "DROP INDEX aggregatedorgunitindicatorvalue_index"; + statementManager.getHolder().executeUpdate( sql, true ); + } + catch ( Exception ex ) + { + log.debug( "Index does not exist" ); + } + } + } + + @Override + public void deleteAggregatedOrgUnitDataValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitdatavalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + statementManager.getHolder().executeUpdate( sql ); + } + + @Override + public void deleteAggregatedOrgUnitIndicatorValues( Collection periodIds ) + { + final String sql = + "DELETE FROM aggregatedorgunitindicatorvalue " + + "WHERE periodid IN ( " + getCommaDelimitedString( periodIds ) + " )"; + + statementManager.getHolder().executeUpdate( sql ); + } +} === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/CrossTabService.java 2012-07-25 16:22:03 +0000 @@ -30,7 +30,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.Future; import org.hisp.dhis.dataelement.DataElementOperand; @@ -46,15 +45,6 @@ { String ID = CrossTabService.class.getName(); - /** - * Filters and returns the DataElementOperands with data from the given - * collection of DataElementOperands. - * - * @param operands the DataElementOperands. - * @return the DataElementOperands with data. - */ - Set getOperandsWithData( Set operands ); - String createCrossTabTable( List operands ); /** === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2012-07-25 16:22:03 +0000 @@ -34,7 +34,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.Future; import org.amplecode.quick.BatchHandler; @@ -45,6 +44,7 @@ import org.apache.commons.logging.LogFactory; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.datamart.CrossTabDataValue; +import org.hisp.dhis.datamart.DataMartManager; import org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore; import org.hisp.dhis.jdbc.batchhandler.GenericBatchHandler; import org.hisp.dhis.organisationunit.OrganisationUnit; @@ -79,22 +79,24 @@ { this.crossTabStore = crossTabTableManager; } - + + private DataMartManager dataMartManager; + + public void setDataMartManager( DataMartManager dataMartManager ) + { + this.dataMartManager = dataMartManager; + } + private StatementManager statementManager; public void setStatementManager( StatementManager statementManager ) { this.statementManager = statementManager; } - + // ------------------------------------------------------------------------- // CrossTabService implementation // ------------------------------------------------------------------------- - - public Set getOperandsWithData( Set operands ) - { - return crossTabStore.getOperandsWithDataValues( operands ); - } public String createCrossTabTable( List operands ) { @@ -121,7 +123,7 @@ { for ( final Integer sourceId : organisationUnitIds ) { - final Map map = crossTabStore.getDataValueMap( periodId, sourceId ); + final Map map = dataMartManager.getDataValueMap( periodId, sourceId ); final List valueList = new ArrayList( operands.size() + 2 ); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/CrossTabStore.java 2012-07-25 16:22:03 +0000 @@ -30,7 +30,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.datamart.CrossTabDataValue; @@ -143,21 +142,4 @@ */ Map getAggregatedOrgUnitDataCacheValue( Collection operands, int periodId, int sourceId, int organisationUnitGroupId, String key ); - - /** - * Gets a set of DataElementOperands which have data values associated. - * - * @param operands the DataElementOperands to check. - * @return a set of DataElementOperands. - */ - Set getOperandsWithDataValues( Set operands ); - - /** - * Gets a Map with entries containing Operand and value for all DataValues registered for the given Period and Source. - * - * @param periodId the Period identifier. - * @param sourceId the Source identifier. - * @return map of data values. - */ - Map getDataValueMap( int periodId, int sourceId ); } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/jdbc/JDBCCrossTabStore.java 2012-07-25 16:22:03 +0000 @@ -34,10 +34,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import org.amplecode.quick.StatementHolder; import org.amplecode.quick.StatementManager; @@ -61,13 +59,6 @@ this.statementManager = statementManager; } - private StatementManager fileDbStatementManager; - - public void setFileDbStatementManager( StatementManager fileDbStatementManager ) - { - this.fileDbStatementManager = fileDbStatementManager; - } - // ------------------------------------------------------------------------- // CrossTabStore implementation // ------------------------------------------------------------------------- @@ -248,69 +239,6 @@ } // ------------------------------------------------------------------------- - // Methods on file database - // ------------------------------------------------------------------------- - - public Set getOperandsWithDataValues( Set operands ) - { - final Set operandsWithData = new HashSet(); - - final StatementHolder holder = fileDbStatementManager.getHolder(); - - for ( DataElementOperand operand : operands ) - { - final String sql = - "SELECT COUNT(*) FROM datavalue " + - "WHERE dataelementid=" + operand.getDataElementId() + " " + - "AND categoryoptioncomboid=" + operand.getOptionComboId(); - - Integer count = holder.queryForInteger( sql ); - - if ( count != null && count > 0 ) - { - operandsWithData.add( operand ); - } - } - - return operandsWithData; - } - - public Map getDataValueMap( int periodId, int sourceId ) - { - final StatementHolder holder = fileDbStatementManager.getHolder(); - - try - { - final String sql = - "SELECT dataelementid, categoryoptioncomboid, value " + - "FROM datavalue " + - "WHERE periodid = " + periodId + " " + - "AND sourceid = " + sourceId; - - final ResultSet resultSet = holder.getStatement().executeQuery( sql ); - - final Map map = new HashMap(); - - while ( resultSet.next() ) - { - final DataElementOperand operand = new DataElementOperand( resultSet.getInt( 1 ), resultSet.getInt( 2 ) ); - - map.put( operand, resultSet.getString( 3 ) ); - } - - return map; - } - catch ( SQLException ex ) - { - throw new RuntimeException( "Failed to get DataValues", ex ); - } - finally - { - holder.close(); - } - } - - // ------------------------------------------------------------------------- // Supportive methods // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-07-12 09:32:15 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2012-07-25 16:22:03 +0000 @@ -27,9 +27,9 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import static org.hisp.dhis.scheduling.TaskCategory.DATAMART; import static org.hisp.dhis.setting.SystemSettingManager.DEFAULT_ORGUNITGROUPSET_AGG_LEVEL; import static org.hisp.dhis.setting.SystemSettingManager.KEY_ORGUNITGROUPSET_AGG_LEVEL; -import static org.hisp.dhis.scheduling.TaskCategory.DATAMART; import static org.hisp.dhis.system.notification.NotificationLevel.INFO; import java.util.ArrayList; @@ -40,14 +40,13 @@ import java.util.Set; import java.util.concurrent.Future; -import org.hisp.dhis.aggregation.AggregatedDataValueService; -import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueService; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementCategoryService; import org.hisp.dhis.dataelement.DataElementOperand; import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.datamart.DataElementOperandList; import org.hisp.dhis.datamart.DataMartEngine; +import org.hisp.dhis.datamart.DataMartManager; import org.hisp.dhis.datamart.crosstab.CrossTabService; import org.hisp.dhis.datamart.dataelement.DataElementDataMart; import org.hisp.dhis.datamart.indicator.IndicatorDataMart; @@ -90,18 +89,11 @@ // Dependencies // ------------------------------------------------------------------------- - private AggregatedDataValueService aggregatedDataValueService; - - public void setAggregatedDataValueService( AggregatedDataValueService aggregatedDataValueService ) - { - this.aggregatedDataValueService = aggregatedDataValueService; - } - - private AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService; - - public void setAggregatedOrgUnitDataValueService( AggregatedOrgUnitDataValueService aggregatedOrgUnitDataValueService ) - { - this.aggregatedOrgUnitDataValueService = aggregatedOrgUnitDataValueService; + private DataMartManager dataMartManager; + + public void setDataMartManager( DataMartManager dataMartManager ) + { + this.dataMartManager = dataMartManager; } private CrossTabService crossTabService; @@ -286,7 +278,7 @@ // Remove operands without data // --------------------------------------------------------------------- - allOperands = crossTabService.getOperandsWithData( allOperands ); + allOperands = dataMartManager.getOperandsWithData( allOperands ); indicatorOperands.retainAll( allOperands ); @@ -330,7 +322,7 @@ // 2. Drop potential indexes // --------------------------------------------------------------------- - aggregatedDataValueService.dropIndex( true, isIndicators ); + dataMartManager.dropAggregatedValueIndex( true, isIndicators ); clock.logTime( "Dropped potential indexes" ); notifier.notify( id, DATAMART, "Deleting existing data element data" ); @@ -339,7 +331,7 @@ // 3. Delete existing aggregated datavalues // --------------------------------------------------------------------- - aggregatedDataValueService.deleteAggregatedDataValues( periodIds ); + dataMartManager.deleteAggregatedDataValues( periodIds ); clock.logTime( "Deleted existing aggregated datavalues" ); notifier.notify( id, DATAMART, "Exporting data for data elements" ); @@ -372,7 +364,7 @@ // 5. Delete existing aggregated indicatorvalues // --------------------------------------------------------------------- - aggregatedDataValueService.deleteAggregatedIndicatorValues( periodIds ); + dataMartManager.deleteAggregatedIndicatorValues( periodIds ); clock.logTime( "Deleted existing aggregated indicatorvalues" ); notifier.notify( id, DATAMART, "Exporting data for indicators" ); @@ -409,7 +401,7 @@ // 8. Create potential indexes // --------------------------------------------------------------------- - aggregatedDataValueService.createIndex( true, isIndicators ); + dataMartManager.createAggregatedValueIndex( true, isIndicators ); clock.logTime( "Created indexes" ); clock.logTime( "Aggregated data export done" ); @@ -433,7 +425,7 @@ // 2. Drop potential indexes // ----------------------------------------------------------------- - aggregatedOrgUnitDataValueService.dropIndex( true, isIndicators ); + dataMartManager.dropAggregatedOrgUnitValueIndex( true, isIndicators ); clock.logTime( "Dropped potential org unit indexes" ); notifier.notify( id, DATAMART, "Deleting existing data element data" ); @@ -442,7 +434,7 @@ // 3. Delete existing aggregated datavalues // --------------------------------------------------------------------- - aggregatedOrgUnitDataValueService.deleteAggregatedDataValues( periodIds ); + dataMartManager.deleteAggregatedOrgUnitDataValues( periodIds ); clock.logTime( "Deleted existing aggregated org unit datavalues" ); notifier.notify( id, DATAMART, "Exporting data for data elements" ); @@ -479,7 +471,7 @@ // 5. Delete existing aggregated indicatorvalues // --------------------------------------------------------------------- - aggregatedOrgUnitDataValueService.deleteAggregatedIndicatorValues( periodIds ); + dataMartManager.deleteAggregatedOrgUnitIndicatorValues( periodIds ); clock.logTime( "Deleted existing aggregated org unit indicatorvalues" ); notifier.notify( id, DATAMART, "Exporting data for indicators" ); @@ -516,7 +508,7 @@ // 8. Create potential indexes // --------------------------------------------------------------------- - aggregatedOrgUnitDataValueService.createIndex( true, isIndicators ); + dataMartManager.createAggregatedOrgUnitValueIndex( true, isIndicators ); clock.logTime( "Created org unit indexes" ); clock.logTime( "Aggregated organisation unit data export done" ); === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-07-25 14:44:02 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2012-07-25 16:22:03 +0000 @@ -8,8 +8,7 @@ - - + @@ -24,16 +23,22 @@ + + + + + + - + === modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java' --- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java 2012-02-07 19:52:19 +0000 +++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/maintenance/PerformMaintenanceAction.java 2012-07-25 16:22:03 +0000 @@ -33,6 +33,7 @@ import org.hisp.dhis.aggregation.AggregatedOrgUnitDataValueService; import org.hisp.dhis.common.DeleteNotAllowedException; import org.hisp.dhis.completeness.DataSetCompletenessService; +import org.hisp.dhis.datamart.DataMartManager; import org.hisp.dhis.maintenance.MaintenanceService; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; @@ -79,6 +80,13 @@ { this.aggregatedOrgUnitDataValueService = aggregatedOrgUnitDataValueService; } + + private DataMartManager dataMartManager; + + public void setDataMartManager( DataMartManager dataMartManager ) + { + this.dataMartManager = dataMartManager; + } private PeriodService periodService; @@ -146,11 +154,11 @@ if ( dataMartIndex ) { - aggregatedDataValueService.dropIndex( true, true ); - aggregatedDataValueService.createIndex( true, true ); + dataMartManager.dropAggregatedValueIndex( true, true ); + dataMartManager.createAggregatedValueIndex( true, true ); - aggregatedOrgUnitDataValueService.dropIndex( true, true ); - aggregatedOrgUnitDataValueService.createIndex( true, true ); + dataMartManager.dropAggregatedOrgUnitValueIndex( true, true ); + dataMartManager.createAggregatedOrgUnitValueIndex( true, true ); completenessService.dropIndex(); completenessService.createIndex(); === modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2012-07-25 13:18:48 +0000 +++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2012-07-25 16:22:03 +0000 @@ -11,8 +11,8 @@ - + +