=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-10 20:41:29 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-11 01:38:43 +0000 @@ -27,18 +27,49 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import java.util.Date; import java.util.List; import java.util.concurrent.Future; public interface AnalyticsTableManager { - void createTable(); - - Future createIndexesAsync( List columns ); - - void swapTable(); - - void populateTable(); + public static final String TABLE_NAME = "analytics"; + public static final String TABLE_TEMP_SUFFIX = "_temp"; + public static final String TABLE_NAME_TEMP = TABLE_NAME + TABLE_TEMP_SUFFIX; + + /** + * Attempts to drop and then create analytics table. + * + * @param tableName the table name. + */ + void createTable( String tableName ); + + /** + * Creates single indexes on the given columns of the analytics table with + * the given name. + * + * @param tableName the name of the table to create indexes on. + * @param columns the columns to create single indexes for. + */ + Future createIndexesAsync( String tableName, List columns ); + + /** + * Attempts to drop analytics table, then rename temporary table to analytics + * table. + * + * @param tableName the name of the analytics table. + */ + void swapTable( String tableName ); + + /** + * Copies and denormalizes rows from data value table into analytics table. + * The data range is based on the start date of the data value row. + * + * @param tableName the name of the analytics table. + * @param startDate the start date for the data value row start date + * @param endDate the end date for the data value row end date + */ + Future populateTableAsync( String tableName, Date startDate, Date endDate ); /** * Returns a list of string arrays in where the first index holds the database @@ -51,4 +82,28 @@ * Returns a list of database column names. */ List getDimensionColumnNames(); + + /** + * Retrieves the start date of the period of the earliest data value row. + */ + Date getEarliestData(); + + /** + * Retrieves the end date of the period of the latest data value row. + */ + Date getLatestData(); + + /** + * Checks whether the given table has no rows, if so drops the table. + * + * @param tableName the name of the table to prune. + */ + void pruneTable( String tableName ); + + /** + * Drops the given table. + * + * @param tableName the name of the table to drop. + */ + void dropTable( String tableName ); } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-10 20:41:29 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-11 01:38:43 +0000 @@ -28,6 +28,7 @@ */ import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.concurrent.Future; @@ -35,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.hisp.dhis.analytics.AnalyticsTableManager; import org.hisp.dhis.analytics.AnalyticsTableService; +import org.hisp.dhis.period.Period; import org.hisp.dhis.system.util.Clock; import org.hisp.dhis.system.util.ConcurrentUtils; import org.hisp.dhis.system.util.PaginatedList; @@ -63,17 +65,29 @@ { Clock clock = new Clock().startClock().logTime( "Starting update..." ); - tableManager.createTable(); - clock.logTime( "Created analytics table" ); - - tableManager.populateTable(); - clock.logTime( "Populated analytics table" ); - - createIndexes(); - clock.logTime( "Created all indexes, update done" ); - - tableManager.swapTable(); - clock.logTime( "Swapped analytics table" ); + final Date earliest = tableManager.getEarliestData(); + final Date latest = tableManager.getLatestData(); + final List tables = ShardUtils.getTempTableNames( earliest, latest ); + clock.logTime( "Checked data timespan" ); + + //dropTables( tables ); //remove + + createTables( tables ); + clock.logTime( "Created analytics tables" ); + + populateTables( tables ); + clock.logTime( "Populated analytics tables" ); + + pruneTables( tables ); + clock.logTime( "Pruned analytics tables" ); + + createIndexes( tables ); + clock.logTime( "Created all indexes" ); + + swapTables( tables ); + clock.logTime( "Swapped analytics tables" ); + + clock.logTime( "Analytics tables update done" ); return null; } @@ -82,21 +96,77 @@ // Supportive methods // ------------------------------------------------------------------------- - private void createIndexes() + private void createTables( List tables ) + { + for ( String table : tables ) + { + tableManager.createTable( table ); + } + } + + private void populateTables( List tables ) + { + int pageSize = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 ); + + List> tablePages = new PaginatedList( tables ).setPageSize( pageSize ).getPages(); + + for ( List tablePage : tablePages ) + { + List> futures = new ArrayList>(); + + for ( String table : tablePage ) + { + Period period = ShardUtils.getPeriod( table ); + + futures.add( tableManager.populateTableAsync( table, period.getStartDate(), period.getEndDate() ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + } + } + + public void pruneTables( List tables ) + { + for ( String table : tables ) + { + tableManager.pruneTable( table ); + } + } + + private void createIndexes( List tables ) { int pages = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 ); log.info( "No of pages: " + pages ); - List> futures = new ArrayList>(); - - List> columnPages = new PaginatedList( tableManager.getDimensionColumnNames() ).setNumberOfPages( pages ).getPages(); - - for ( List columnPage : columnPages ) - { - futures.add( tableManager.createIndexesAsync( columnPage ) ); - } - - ConcurrentUtils.waitForCompletion( futures ); + for ( String table : tables ) + { + List> futures = new ArrayList>(); + + List> columnPages = new PaginatedList( tableManager.getDimensionColumnNames() ).setNumberOfPages( pages ).getPages(); + + for ( List columnPage : columnPages ) + { + futures.add( tableManager.createIndexesAsync( table, columnPage ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + } + } + + private void swapTables( List tables ) + { + for ( String table : tables ) + { + tableManager.swapTable( table ); + } + } + + protected void dropTables( List tables ) + { + for ( String table : tables ) + { + tableManager.dropTable( table ); + } } } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-10 20:41:29 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-11 01:38:43 +0000 @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Date; import java.util.List; import java.util.concurrent.Future; @@ -41,6 +42,7 @@ import org.hisp.dhis.organisationunit.OrganisationUnitLevel; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.system.util.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; @@ -62,12 +64,10 @@ implements AnalyticsTableManager { private static final Log log = LogFactory.getLog( JdbcAnalyticsTableManager.class ); - + public static final String PREFIX_ORGUNITGROUPSET = "ougs_"; public static final String PREFIX_ORGUNITLEVEL = "idlevel"; public static final String PREFIX_INDEX = "index_"; - public static final String TABLE_NAME = "analytics"; - public static final String TABLE_NAME_TEMP = "analytics_temp"; @Autowired private OrganisationUnitService organisationUnitService; @@ -84,14 +84,14 @@ //TODO shard on data quarter based on start date //TODO average aggregation operator data, pre-aggregate in time dimension, not in org unit dimension - - public void createTable() + + public void createTable( String tableName ) { - final String sqlDrop = "drop table " + TABLE_NAME_TEMP; + final String sqlDrop = "drop table " + tableName; executeSilently( sqlDrop ); - String sqlCreate = "create table " + TABLE_NAME_TEMP + " ("; + String sqlCreate = "create table " + tableName + " ("; for ( String[] col : getDimensionColumns() ) { @@ -106,16 +106,17 @@ } @Async - public Future createIndexesAsync( List columns ) + public Future createIndexesAsync( String tableName, List columns ) { for ( String column : columns ) { - final String sql = "create index " + PREFIX_INDEX + - column + " on " + TABLE_NAME_TEMP + " (" + column + ")"; + final String index = PREFIX_INDEX + column + "_" + tableName; + + final String sql = "create index " + index + " on " + tableName + " (" + column + ")"; executeSilently( sql ); - log.info( "Created index on column: " + column ); + log.info( "Created index: " + index ); } log.info( "Indexes created" ); @@ -123,27 +124,35 @@ return null; } - public void swapTable() + public void swapTable( String tableName ) { - final String sqlDrop = "drop table " + TABLE_NAME; + final String realTable = tableName.replaceFirst( TABLE_TEMP_SUFFIX, "" ); + + final String sqlDrop = "drop table " + realTable; executeSilently( sqlDrop ); - final String sqlAlter = "alter table " + TABLE_NAME_TEMP + " rename to " + TABLE_NAME; - - jdbcTemplate.execute( sqlAlter ); - } - - public void populateTable() - { - populateTable( "cast(dv.value as double precision)" , "int" ); - - populateTable( "1 as value" , "bool" ); - } - - private void populateTable( String valueExpression, String valueType ) - { - String insert = "insert into " + TABLE_NAME_TEMP + " ("; + final String sqlAlter = "alter table " + tableName + " rename to " + realTable; + + executeSilently( sqlAlter ); + } + + @Async + public Future populateTableAsync( String tableName, Date startDate, Date endDate ) + { + populateTable( tableName, startDate, endDate, "cast(dv.value as double precision)", "int" ); + + populateTable( tableName, startDate, endDate, "1 as value" , "bool" ); + + return null; + } + + private void populateTable( String tableName, Date startDate, Date endDate, String valueExpression, String valueType ) + { + final String start = DateUtils.getMediumDateString( startDate ); + final String end = DateUtils.getMediumDateString( endDate ); + + String insert = "insert into " + tableName + " ("; for ( String[] col : getDimensionColumns() ) { @@ -168,7 +177,9 @@ "left join _period_no_disaggregation_structure ps on dv.periodid=ps.periodid " + "left join dataelement de on dv.dataelementid=de.dataelementid " + "left join period pe on dv.periodid=pe.periodid " + - "where de.valuetype='" + valueType + "' and pe.startdate >= '2011-10-01'"; + "where de.valuetype='" + valueType + "' " + + "and pe.startdate >= '" + start + "' " + + "and pe.startdate <= '" + end + "'"; final String sql = insert + select; @@ -229,6 +240,48 @@ return columnNames; } + public Date getEarliestData() + { + final String sql = "select min(pe.startdate) from datavalue dv " + + "join period pe on dv.periodid=pe.periodid"; + + return jdbcTemplate.queryForObject( sql, Date.class ); + } + + public Date getLatestData() + { + final String sql = "select max(pe.startdate) from datavalue dv " + + "join period pe on dv.periodid=pe.periodid"; + + return jdbcTemplate.queryForObject( sql, Date.class ); + } + + public void pruneTable( String tableName ) + { + final String sqlCount = "select count(*) from " + tableName; + + log.info( "Count SQL: " + sqlCount ); + + final boolean empty = jdbcTemplate.queryForInt( sqlCount ) == 0; + + if ( empty ) + { + final String sqlDrop = "drop table " + tableName; + + executeSilently( sqlDrop ); + + log.info( "Drop SQL: " + sqlDrop ); + } + } + + public void dropTable( String tableName ) + { + final String realTable = tableName.replaceFirst( TABLE_TEMP_SUFFIX, "" ); + + executeSilently( "drop table " + tableName ); + executeSilently( "drop table " + realTable ); + } + // ------------------------------------------------------------------------- // Supportive methods // ------------------------------------------------------------------------- === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ShardUtils.java 2012-12-11 01:38:43 +0000 @@ -0,0 +1,96 @@ +package org.hisp.dhis.analytics.table; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.analytics.AnalyticsTableManager.TABLE_NAME; +import static org.hisp.dhis.analytics.AnalyticsTableManager.TABLE_NAME_TEMP; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.period.QuarterlyPeriodType; + +public class ShardUtils +{ + private static final QuarterlyPeriodType QUARTERLY = new QuarterlyPeriodType(); + + private static final String SEP = "_"; + + public static List getTempTableNames( Date earliest, Date latest ) + { + if ( earliest == null || latest == null || earliest.after( latest ) ) + { + throw new IllegalArgumentException( "Earliest or latest date invalid: " + earliest + ", " + latest ); + } + + List tables = new ArrayList(); + + Period period = QUARTERLY.createPeriod( earliest ); + + while ( period != null && period.getStartDate().before( latest ) ) + { + String table = TABLE_NAME_TEMP + SEP + period.getIsoDate(); + + tables.add( table ); + + period = QUARTERLY.getNextPeriod( period ); + } + + return tables; + } + + public static String getTable( String isoPeriod ) + { + Period period = PeriodType.getPeriodFromIsoString( isoPeriod ); + + if ( period == null ) + { + throw new IllegalArgumentException( "Illegal ISO period: " + isoPeriod ); + } + + Period quarter = QUARTERLY.createPeriod( period.getStartDate() ); + + return TABLE_NAME + SEP + quarter.getIsoDate(); + } + + public static Period getPeriod( String tableName ) + { + if ( tableName == null || tableName.indexOf( SEP ) == -1 ) + { + throw new IllegalArgumentException( "Illegal table name: " + tableName ); + } + + String[] split = tableName.split( SEP ); + String isoPeriod = split[split.length - 1]; + + return PeriodType.getPeriodFromIsoString( isoPeriod ); + } +} === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java' --- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/ShardUtilsTest.java 2012-12-11 01:38:43 +0000 @@ -0,0 +1,82 @@ +package org.hisp.dhis.analytics.table; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.analytics.AnalyticsTableManager.TABLE_NAME; +import static org.hisp.dhis.analytics.AnalyticsTableManager.TABLE_NAME_TEMP; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Date; +import java.util.List; + +import org.hisp.dhis.period.Cal; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.QuarterlyPeriodType; +import org.junit.Test; + +public class ShardUtilsTest +{ + @Test + public void testGetTableNames() + { + Cal cal = new Cal(); + Date earliest = cal.set( 2000, 5, 4 ).time(); + Date latest = cal.set( 2001, 2, 10 ).time(); + + List tables = ShardUtils.getTempTableNames( earliest, latest ); + + assertEquals( 4, tables.size() ); + assertTrue( tables.contains( TABLE_NAME_TEMP + "_2000Q2" ) ); + assertTrue( tables.contains( TABLE_NAME_TEMP + "_2000Q3" ) ); + assertTrue( tables.contains( TABLE_NAME_TEMP + "_2000Q4" ) ); + assertTrue( tables.contains( TABLE_NAME_TEMP + "_2001Q1" ) ); + } + + @Test + public void testGetTable() + { + assertEquals( TABLE_NAME + "_2000Q4", ShardUtils.getTable( "200011" ) ); + assertEquals( TABLE_NAME + "_2000Q1", ShardUtils.getTable( "2000W02" ) ); + assertEquals( TABLE_NAME + "_2000Q2", ShardUtils.getTable( "2000Q2" ) ); + assertEquals( TABLE_NAME + "_2000Q3", ShardUtils.getTable( "2000S2" ) ); + assertEquals( TABLE_NAME + "_2000Q1", ShardUtils.getTable( "2000" ) ); + } + + @Test + public void testGetPeriod() + { + Cal cal = new Cal(); + + Period q2 = new QuarterlyPeriodType().createPeriod( cal.set( 2000, 4, 1 ).time() ); + Period q4 = new QuarterlyPeriodType().createPeriod( cal.set( 2000, 10, 1 ).time() ); + + assertEquals( q2, ShardUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q2" ) ); + assertEquals( q4, ShardUtils.getPeriod( TABLE_NAME_TEMP + "_2000Q4" ) ); + } +}