=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2010-10-22 06:43:14 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/crosstab/DefaultCrossTabService.java 2010-10-22 12:37:35 +0000 @@ -49,155 +49,159 @@ /** * @author Lars Helge Overland - * @version $Id: DefaultCrossTabService.java 6268 2008-11-12 15:16:02Z larshelg $ + * @version $Id: DefaultCrossTabService.java 6268 2008-11-12 15:16:02Z larshelg + * $ */ public class DefaultCrossTabService implements CrossTabService { private static final Log log = LogFactory.getLog( DefaultCrossTabService.class ); - + private static final int MAX_LENGTH = 20; - + // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- @Autowired private BatchHandlerFactory batchHandlerFactory; - + private CrossTabStore crossTabStore; - + public void setCrossTabStore( CrossTabStore crossTabTableManager ) { this.crossTabStore = crossTabTableManager; } - + private AggregatedDataValueService aggregatedDataValueService; public void setAggregatedDataValueService( AggregatedDataValueService aggregatedDataValueService ) { this.aggregatedDataValueService = aggregatedDataValueService; } - + // ------------------------------------------------------------------------- // CrossTabService implementation // ------------------------------------------------------------------------- - public Collection populateCrossTabTable( final Collection operands, + public Collection populateCrossTabTable( final Collection operands, final Collection periodIds, final Collection organisationUnitIds, String key ) { - final Set operandsWithData = new HashSet( operands ); - if ( validate( operands, periodIds, organisationUnitIds ) ) { + final Set operandsWithData = new HashSet( operands ); + final List operandList = new ArrayList( operands ); - + Collections.sort( operandList ); crossTabStore.dropCrossTabTable( key ); - + log.info( "Dropped crosstab table" ); - + crossTabStore.createCrossTabTable( operandList, key ); - + log.info( "Created crosstab table" ); - System.out.println("\n\n ================= \n create table name : " + CrossTabStore.TABLE_NAME + key); - final BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( GenericBatchHandler.class ); + + final BatchHandler batchHandler = batchHandlerFactory + .createBatchHandler( GenericBatchHandler.class ); batchHandler.setTableName( CrossTabStore.TABLE_NAME + key ); batchHandler.init(); - + Map map = null; - + List valueList = null; - + boolean hasValues = false; - + String value = null; - + for ( final Integer periodId : periodIds ) { for ( final Integer sourceId : organisationUnitIds ) { map = aggregatedDataValueService.getDataValueMap( periodId, sourceId ); - + valueList = new ArrayList( operandList.size() + 2 ); - + valueList.add( String.valueOf( periodId ) ); valueList.add( String.valueOf( sourceId ) ); hasValues = false; - + for ( DataElementOperand operand : operandList ) { value = map.get( operand ); - + if ( value != null && value.length() > MAX_LENGTH ) { - log.warn( "Value ignored, too long: '" + value + - "', for dataelement id: '" + operand.getDataElementId() + - "', categoryoptioncombo id: '" + operand.getOptionComboId() + - "', period id: '" + periodId + - "', source id: '" + sourceId + "'" ); - + log.warn( "Value ignored, too long: '" + value + "', for dataelement id: '" + + operand.getDataElementId() + "', categoryoptioncombo id: '" + + operand.getOptionComboId() + "', period id: '" + periodId + "', source id: '" + + sourceId + "'" ); + value = null; - } - + } + if ( value != null ) { - hasValues = true; - operandsWithData.add( operand ); + hasValues = true; + operandsWithData.add( operand ); } - + valueList.add( value ); } - + if ( hasValues ) { batchHandler.addObject( valueList ); } } - + log.info( "Crosstabulated data for period " + periodId ); } - + batchHandler.flush(); + + return operandsWithData; } - return operandsWithData; + return null; + } - + public void dropCrossTabTable( String key ) { crossTabStore.dropCrossTabTable( key ); } - + public void trimCrossTabTable( Collection operands, String key ) { // TODO use H2 in-memory table for datavaluecrosstab table ? - + crossTabStore.createTrimmedCrossTabTable( operands, key ); - + crossTabStore.dropCrossTabTable( key ); - + crossTabStore.renameTrimmedCrossTabTable( key ); } - + public Map getOperandIndexMap( Collection operands, String key ) { final Map columnNameIndexMap = crossTabStore.getCrossTabTableColumns( key ); final Map operandMap = new HashMap(); - + for ( DataElementOperand operand : operands ) { final String col = operand.getSimpleName(); - + if ( columnNameIndexMap.containsKey( col ) ) { operandMap.put( operand, columnNameIndexMap.get( col ) ); } } - + return operandMap; } @@ -205,44 +209,48 @@ { return crossTabStore.validateCrossTabTable( operands ); } - - public Collection getCrossTabDataValues( Map operandIndexMap, Collection periodIds, Collection sourceIds, String key ) + + public Collection getCrossTabDataValues( Map operandIndexMap, + Collection periodIds, Collection sourceIds, String key ) { return crossTabStore.getCrossTabDataValues( operandIndexMap, periodIds, sourceIds, key ); } - - public Collection getCrossTabDataValues( Map operandIndexMap, Collection periodIds, int sourceId, String key ) + + public Collection getCrossTabDataValues( Map operandIndexMap, + Collection periodIds, int sourceId, String key ) { return crossTabStore.getCrossTabDataValues( operandIndexMap, periodIds, sourceId, key ); } - + // ------------------------------------------------------------------------- // Supportive methods // ------------------------------------------------------------------------- /** - * Validates whether the given collections of identifiers are not null and of size greater than 0. + * Validates whether the given collections of identifiers are not null and + * of size greater than 0. */ - private boolean validate( Collection operands, Collection periodIds, Collection unitIds ) + private boolean validate( Collection operands, Collection periodIds, + Collection unitIds ) { if ( operands == null || operands.size() == 0 ) { - log.warn( "No operands selected for crosstab table" ); + log.warn( "No operands selected for crosstab table" ); return false; } - + if ( periodIds == null || periodIds.size() == 0 ) { - log.warn( "No periods selected for crosstab table" ); + log.warn( "No periods selected for crosstab table" ); return false; } - + if ( unitIds == null || unitIds.size() == 0 ) { log.warn( "No organisation units selected for crosstab table" ); return false; } - + return true; } } === modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java' --- dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-08-31 05:47:11 +0000 +++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/java/org/hisp/dhis/datamart/engine/DefaultDataMartEngine.java 2010-10-22 12:37:35 +0000 @@ -71,13 +71,13 @@ implements DataMartEngine { private static final Log log = LogFactory.getLog( DefaultDataMartEngine.class ); - + // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- protected AggregationCache aggregationCache; - + public void setAggregationCache( AggregationCache aggregationCache ) { this.aggregationCache = aggregationCache; @@ -89,7 +89,7 @@ { this.aggregatedDataValueService = aggregatedDataValueService; } - + private CrossTabService crossTabService; public void setCrossTabService( CrossTabService crossTabService ) @@ -110,14 +110,14 @@ { this.indicatorDataMart = indicatorDataMart; } - + private CalculatedDataElementDataMart calculatedDataElementDataMart; public void setCalculatedDataElementDataMart( CalculatedDataElementDataMart calculatedDataElementDataMart ) { this.calculatedDataElementDataMart = calculatedDataElementDataMart; } - + private DataElementAggregator sumIntAggregator; public void setSumIntAggregator( DataElementAggregator sumIntDataElementAggregator ) @@ -133,7 +133,7 @@ } private DataElementAggregator averageIntSingleValueAggregator; - + public void setAverageIntSingleValueAggregator( DataElementAggregator averageIntSingleValueAggregator ) { this.averageIntSingleValueAggregator = averageIntSingleValueAggregator; @@ -159,14 +159,14 @@ { this.dataElementService = dataElementService; } - + private IndicatorService indicatorService; public void setIndicatorService( IndicatorService indicatorService ) { this.indicatorService = indicatorService; } - + private PeriodService periodService; public void setPeriodService( PeriodService periodService ) @@ -187,7 +187,7 @@ { this.expressionService = expressionService; } - + private OrganisationUnitService organisationUnitService; public void setOrganisationUnitService( OrganisationUnitService organisationUnitService ) @@ -202,9 +202,9 @@ @Transactional public int export( Collection dataElementIds, Collection indicatorIds, Collection periodIds, Collection organisationUnitIds, ProcessState state ) - { + { int count = 0; - + TimeUtils.start(); state.setMessage( "deleting_existing_aggregated_data" ); @@ -214,9 +214,9 @@ // --------------------------------------------------------------------- aggregatedDataValueService.deleteAggregatedDataValues( dataElementIds, periodIds, organisationUnitIds ); - + aggregatedDataValueService.deleteAggregatedIndicatorValues( indicatorIds, periodIds, organisationUnitIds ); - + log.info( "Deleted existing aggregated data: " + TimeUtils.getHMS() ); // --------------------------------------------------------------------- @@ -225,54 +225,69 @@ final Set nonCalculatedDataElementIds = filterCalculatedDataElementIds( dataElementIds, false ); final Set calculatedDataElementIds = filterCalculatedDataElementIds( dataElementIds, true ); - + final Set dataElementInIndicatorIds = getDataElementIdsInIndicators( indicatorIds ); final Set dataElementInCalculatedDataElementIds = getDataElementIdsInCalculatedDataElements( calculatedDataElementIds ); - + final Set allDataElementIds = new HashSet(); allDataElementIds.addAll( nonCalculatedDataElementIds ); allDataElementIds.addAll( dataElementInIndicatorIds ); allDataElementIds.addAll( dataElementInCalculatedDataElementIds ); - final Collection allDataElementOperands = categoryService.getOperandsByIds( allDataElementIds ); - final Collection dataElementInIndicatorOperands = categoryService.getOperandsByIds( dataElementInIndicatorIds ); - final Collection dataElementInCalculatedDataElementOperands = categoryService.getOperandsByIds( dataElementInCalculatedDataElementIds ); - final Collection nonCalculatedDataElementOperands = categoryService.getOperandsByIds( nonCalculatedDataElementIds ); - - final Collection sumIntDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM ); - final Collection averageIntDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE ); - final Collection sumBoolDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_SUM ); - final Collection averageBoolDataElementOperands = filterOperands( nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_AVERAGE ); - + final Collection allDataElementOperands = categoryService + .getOperandsByIds( allDataElementIds ); + final Collection dataElementInIndicatorOperands = categoryService + .getOperandsByIds( dataElementInIndicatorIds ); + final Collection dataElementInCalculatedDataElementOperands = categoryService + .getOperandsByIds( dataElementInCalculatedDataElementIds ); + final Collection nonCalculatedDataElementOperands = categoryService + .getOperandsByIds( nonCalculatedDataElementIds ); + + final Collection sumIntDataElementOperands = filterOperands( + nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_SUM ); + final Collection averageIntDataElementOperands = filterOperands( + nonCalculatedDataElementOperands, VALUE_TYPE_INT, AGGREGATION_OPERATOR_AVERAGE ); + final Collection sumBoolDataElementOperands = filterOperands( + nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_SUM ); + final Collection averageBoolDataElementOperands = filterOperands( + nonCalculatedDataElementOperands, VALUE_TYPE_BOOL, AGGREGATION_OPERATOR_AVERAGE ); + log.info( "Filtered data elements: " + TimeUtils.getHMS() ); - + // --------------------------------------------------------------------- // Create and trim crosstabtable // --------------------------------------------------------------------- String key = RandomStringUtils.randomAlphanumeric( 8 ); - + if ( crossTabService.validateCrossTabTable( allDataElementOperands ) != 0 ) { int excess = crossTabService.validateCrossTabTable( allDataElementOperands ); log.warn( "Cannot crosstabulate since the number of data elements exceeded maximum columns: " + excess ); - + state.setMessage( "could_not_export_too_many_data_elements" ); - + return 0; - } + } log.info( "Validated crosstab table: " + TimeUtils.getHMS() ); - + state.setMessage( "crosstabulating_data" ); - Collection childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( organisationUnitIds ); + Collection childrenIds = organisationUnitService.getOrganisationUnitHierarchy().getChildren( + organisationUnitIds ); + + final Collection emptyOperands = crossTabService.populateCrossTabTable( + allDataElementOperands, getIntersectingIds( periodIds ), childrenIds, key ); - final Collection emptyOperands = crossTabService.populateCrossTabTable( allDataElementOperands, getIntersectingIds( periodIds ), childrenIds, key ); - log.info( "Populated crosstab table: " + TimeUtils.getHMS() ); - + + if ( emptyOperands == null ) + { + return 0; + } + crossTabService.trimCrossTabTable( emptyOperands, key ); log.info( "Trimmed crosstab table: " + TimeUtils.getHMS() ); @@ -285,37 +300,48 @@ if ( sumIntDataElementOperands.size() > 0 ) { - count += dataElementDataMart.exportDataValues( sumIntDataElementOperands, periodIds, organisationUnitIds, sumIntAggregator, key ); - - log.info( "Exported values for data element operands with sum aggregation operator of type number (" + sumIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); - } - - if ( averageIntDataElementOperands.size() > 0 ) - { - count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, organisationUnitIds, averageIntAggregator, key ); - - log.info( "Exported values for data element operands with average aggregation operator of type number (" + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); - } - - if ( averageIntDataElementOperands.size() > 0 ) - { - count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, organisationUnitIds, averageIntSingleValueAggregator, key ); - - log.info( "Exported values for data element operands with average aggregation operator with single value of type number (" + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); + count += dataElementDataMart.exportDataValues( sumIntDataElementOperands, periodIds, organisationUnitIds, + sumIntAggregator, key ); + + log.info( "Exported values for data element operands with sum aggregation operator of type number (" + + sumIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); + } + + if ( averageIntDataElementOperands.size() > 0 ) + { + count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, + organisationUnitIds, averageIntAggregator, key ); + + log.info( "Exported values for data element operands with average aggregation operator of type number (" + + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); + } + + if ( averageIntDataElementOperands.size() > 0 ) + { + count += dataElementDataMart.exportDataValues( averageIntDataElementOperands, periodIds, + organisationUnitIds, averageIntSingleValueAggregator, key ); + + log + .info( "Exported values for data element operands with average aggregation operator with single value of type number (" + + averageIntDataElementOperands.size() + "): " + TimeUtils.getHMS() ); } if ( sumBoolDataElementOperands.size() > 0 ) { - count += dataElementDataMart.exportDataValues( sumBoolDataElementOperands, periodIds, organisationUnitIds, sumBoolAggregator, key ); - - log.info( "Exported values for data element operands with sum aggregation operator of type yes/no (" + sumBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() ); + count += dataElementDataMart.exportDataValues( sumBoolDataElementOperands, periodIds, organisationUnitIds, + sumBoolAggregator, key ); + + log.info( "Exported values for data element operands with sum aggregation operator of type yes/no (" + + sumBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() ); } if ( averageBoolDataElementOperands.size() > 0 ) { - count += dataElementDataMart.exportDataValues( averageBoolDataElementOperands, periodIds, organisationUnitIds, averageBoolAggregator, key ); - - log.info( "Exported values for data element operands with average aggregation operator of type yes/no (" + averageBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() ); + count += dataElementDataMart.exportDataValues( averageBoolDataElementOperands, periodIds, + organisationUnitIds, averageBoolAggregator, key ); + + log.info( "Exported values for data element operands with average aggregation operator of type yes/no (" + + averageBoolDataElementOperands.size() + "): " + TimeUtils.getHMS() ); } state.setMessage( "exporting_data_for_indicators" ); @@ -326,8 +352,9 @@ if ( indicatorIds != null && indicatorIds.size() > 0 ) { - count += indicatorDataMart.exportIndicatorValues( indicatorIds, periodIds, organisationUnitIds, dataElementInIndicatorOperands, key ); - + count += indicatorDataMart.exportIndicatorValues( indicatorIds, periodIds, organisationUnitIds, + dataElementInIndicatorOperands, key ); + log.info( "Exported values for indicators (" + indicatorIds.size() + "): " + TimeUtils.getHMS() ); } @@ -339,19 +366,21 @@ if ( calculatedDataElementIds != null && calculatedDataElementIds.size() > 0 ) { - count += calculatedDataElementDataMart.exportCalculatedDataElements( calculatedDataElementIds, periodIds, organisationUnitIds, dataElementInCalculatedDataElementOperands, key ); - - log.info( "Exported values for calculated data elements (" + calculatedDataElementIds.size() + "): " + TimeUtils.getHMS() ); + count += calculatedDataElementDataMart.exportCalculatedDataElements( calculatedDataElementIds, periodIds, + organisationUnitIds, dataElementInCalculatedDataElementOperands, key ); + + log.info( "Exported values for calculated data elements (" + calculatedDataElementIds.size() + "): " + + TimeUtils.getHMS() ); } crossTabService.dropCrossTabTable( key ); - + log.info( "Export process completed: " + TimeUtils.getHMS() ); - + TimeUtils.stop(); aggregationCache.clearCache(); - + return count; } @@ -360,101 +389,104 @@ // ------------------------------------------------------------------------- /** - * Sorts calculated data elements from non-calculated based on the given collection - * of identifiers and flag. + * Sorts calculated data elements from non-calculated based on the given + * collection of identifiers and flag. */ private Set filterCalculatedDataElementIds( final Collection dataElementIds, boolean calculated ) { final Set identifiers = new HashSet(); - + for ( final Integer id : dataElementIds ) { final DataElement element = dataElementService.getDataElement( id ); - - if ( ( element instanceof CalculatedDataElement ) == calculated ) + + if ( (element instanceof CalculatedDataElement) == calculated ) { identifiers.add( id ); } } - + return identifiers; } - + /** - * Returns all data element identifiers included in the indicators in the given - * identifier collection. + * Returns all data element identifiers included in the indicators in the + * given identifier collection. */ private Set getDataElementIdsInIndicators( final Collection indicatorIds ) { final Set identifiers = new HashSet( indicatorIds.size() ); - + for ( final Integer id : indicatorIds ) { final Indicator indicator = indicatorService.getIndicator( id ); - + identifiers.addAll( getDataElementIdsInExpression( indicator.getNumerator() ) ); - identifiers.addAll( getDataElementIdsInExpression( indicator.getDenominator() ) ); + identifiers.addAll( getDataElementIdsInExpression( indicator.getDenominator() ) ); } - + return identifiers; } - + /** - * Returns all data element identifiers included in the calculated data elements - * in the given identifier collection. + * Returns all data element identifiers included in the calculated data + * elements in the given identifier collection. */ private Set getDataElementIdsInCalculatedDataElements( final Collection calculatedDataElementIds ) { final Set identifiers = new HashSet(); - + for ( final Integer id : calculatedDataElementIds ) { final Set dataElements = expressionService.getDataElementsInCalculatedDataElement( id ); - + if ( dataElements != null ) { identifiers.addAll( ConversionUtils.getIdentifiers( DataElement.class, dataElements ) ); } } - + return identifiers; } - + /** - * Returns the identifiers of the periods in the given collection including + * Returns the identifiers of the periods in the given collection including * all intersecting periods. */ private Collection getIntersectingIds( final Collection periodIds ) { final Set identifiers = new HashSet( periodIds.size() ); - + for ( final Integer id : periodIds ) { final Period period = periodService.getPeriod( id ); - - final Collection periods = periodService.getIntersectingPeriods( period.getStartDate(), period.getEndDate() ); - + + final Collection periods = periodService.getIntersectingPeriods( period.getStartDate(), period + .getEndDate() ); + identifiers.addAll( ConversionUtils.getIdentifiers( Period.class, periods ) ); } - + return identifiers; } - + /** * Filters the data element operands based on the value type. */ - private Collection filterOperands( Collection operands, String valueType, String aggregationOperator ) + private Collection filterOperands( Collection operands, String valueType, + String aggregationOperator ) { final Collection filtered = new ArrayList(); - + for ( DataElementOperand operand : operands ) { - if ( operand.getValueType().equals( valueType ) && operand.getAggregationOperator().equals( aggregationOperator ) ) + if ( operand.getValueType().equals( valueType ) + && operand.getAggregationOperator().equals( aggregationOperator ) ) { filtered.add( operand ); } } - + return filtered; } }