=== modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataElementConverter.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataElementConverter.java 2012-03-22 13:09:53 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataElementConverter.java 2012-06-27 07:16:03 +0000 @@ -56,10 +56,10 @@ * @version $Id: DataElementConverter.java 6455 2008-11-24 08:59:37Z larshelg $ */ public class DataElementConverter - extends DataElementImporter implements XMLConverter + extends DataElementImporter + implements XMLConverter { public static final String ELEMENT_NAME = "DataElement"; - private static final String FIELD_ID = "DataElementID"; private static final String FIELD_SORT_ORDER = "SortOrder"; private static final String FIELD_CODE = "DataElementCode"; @@ -81,7 +81,6 @@ private static final String FIELD_SELECTED = "Selected"; private static final String FIELD_LAST_USER = "LastUserID"; private static final String FIELD_LAST_UPDATED = "LastUpdated"; - private static final int VALID_FROM = 34335; private static final int VALID_TO = 2958465; private static final int AGG_START_LEVEL = 5; @@ -91,7 +90,7 @@ // ------------------------------------------------------------------------- private DataElementCategoryService categoryService; - + // ------------------------------------------------------------------------- // Constructor // ------------------------------------------------------------------------- @@ -110,34 +109,33 @@ * @param importObjectService the importObjectService to use. * @param dataElementService the dataElementService to use. */ - public DataElementConverter( ImportObjectService importObjectService, - DataElementService dataElementService, - DataElementCategoryService categoryService, - ImportAnalyser importAnalyser ) + public DataElementConverter( ImportObjectService importObjectService, DataElementService dataElementService, + DataElementCategoryService categoryService, ImportAnalyser importAnalyser ) { this.importObjectService = importObjectService; this.dataElementService = dataElementService; this.categoryService = categoryService; this.importAnalyser = importAnalyser; } - + // ------------------------------------------------------------------------- // XMLConverter implementation // ------------------------------------------------------------------------- - + public void write( XMLWriter writer, ExportParams params ) { - Collection elements = dataElementService.getDataElements( params.getDataElements() ); // Calculated elements handled together - + Collection elements = dataElementService.getDataElements( params.getDataElements() ); + if ( elements != null && elements.size() > 0 ) { int i = 0; for ( DataElement object : elements ) { writer.openElement( ELEMENT_NAME ); - + writer.writeElement( FIELD_ID, String.valueOf( object.getId() ) ); - writer.writeElement( FIELD_SORT_ORDER, object.getSortOrder() != null ? String.valueOf( object.getSortOrder() ) : String.valueOf( i++ ) ); + writer.writeElement( FIELD_SORT_ORDER, + object.getSortOrder() != null ? String.valueOf( object.getSortOrder() ) : String.valueOf( i++ ) ); writer.writeElement( FIELD_CODE, object.getCode() ); writer.writeElement( FIELD_NAME, object.getName() ); writer.writeElement( FIELD_SHORT_NAME, object.getShortName() ); @@ -153,40 +151,45 @@ writer.writeElement( FIELD_CALCULATED, convertBooleanToDhis14( false ) ); writer.writeElement( FIELD_SAVE_CALCULATED, convertBooleanToDhis14( false ) ); writer.writeElement( FIELD_AGGREGATION_START_LEVEL, String.valueOf( AGG_START_LEVEL ) ); - writer.writeElement( FIELD_AGGREGATION_OPERATOR, convertAggregationOperatorToDhis14( object.getAggregationOperator() ) ); + writer.writeElement( FIELD_AGGREGATION_OPERATOR, + convertAggregationOperatorToDhis14( object.getAggregationOperator() ) ); writer.writeElement( FIELD_SELECTED, String.valueOf( 0 ) ); writer.writeElement( FIELD_LAST_USER, String.valueOf( 1 ) ); writer.writeElement( FIELD_LAST_UPDATED, Dhis14DateUtil.getDateString( object.getLastUpdated() ) ); - + writer.closeElement(); - - NameMappingUtil.addDataElementAggregationOperatorMapping( object.getId(), object.getAggregationOperator() ); + + NameMappingUtil.addDataElementAggregationOperatorMapping( object.getId(), + object.getAggregationOperator() ); } } } - + public void read( XMLReader reader, ImportParams params ) { Map values = reader.readElements( ELEMENT_NAME ); - - DataElementCategoryCombo categoryCombo = categoryService.getDataElementCategoryComboByName( DEFAULT_CATEGORY_COMBO_NAME ); + + DataElementCategoryCombo categoryCombo = categoryService + .getDataElementCategoryComboByName( DEFAULT_CATEGORY_COMBO_NAME ); DataElementCategoryCombo proxyCategoryCombo = new DataElementCategoryCombo(); proxyCategoryCombo.setId( categoryCombo.getId() ); - + DataElement element = new DataElement(); - + element.setCategoryCombo( proxyCategoryCombo ); element.setId( Integer.valueOf( values.get( FIELD_ID ) ) ); element.setName( values.get( FIELD_NAME ) ); element.setShortName( values.get( FIELD_SHORT_NAME ) ); element.setDescription( Dhis14ParsingUtils.removeNewLine( values.get( FIELD_DESCRIPTION ) ) ); - element.setAlternativeName(values.get(FIELD_PROMPT)); - element.setActive( true ); - element.setType( Dhis14ObjectMappingUtil.getDataElementTypeMap().get( Integer.parseInt( values.get( FIELD_DATA_TYPE ) ) ) ); + element.setAlternativeName( values.get( FIELD_PROMPT ) ); + element.setActive( true ); + element.setType( Dhis14ObjectMappingUtil.getDataElementTypeMap().get(Integer.parseInt( values.get( FIELD_DATA_TYPE ) ) ) ); element.setAggregationOperator( convertAggregationOperatorFromDhis14( values.get( FIELD_AGGREGATION_OPERATOR ) ) ); element.setSortOrder( parseInt( values.get( FIELD_SORT_ORDER ) ) ); element.setLastUpdated( Dhis14DateUtil.getDate( values.get( FIELD_LAST_UPDATED ) ) ); - - importObject( element, params ); - } + if ( values.get( FIELD_CALCULATED ).equals( "0" ) ) //Ignore calculated data elements + { + importObject( element, params ); + } + } } === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java 2012-06-26 21:04:37 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/converter/DataValueConverter.java 2012-06-27 07:16:03 +0000 @@ -27,56 +27,72 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import static org.hisp.dhis.system.util.CsvUtils.NEWLINE; +import static org.hisp.dhis.system.util.CsvUtils.SEPARATOR_B; +import static org.hisp.dhis.system.util.CsvUtils.csvEncode; +import static org.hisp.dhis.system.util.CsvUtils.getCsvEndValue; +import static org.hisp.dhis.system.util.CsvUtils.getCsvValue; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + import org.amplecode.quick.BatchHandler; import org.amplecode.quick.StatementManager; import org.hisp.dhis.aggregation.AggregatedDataValueService; import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo; import org.hisp.dhis.dataelement.DataElementCategoryService; -import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.datavalue.DataValue; import org.hisp.dhis.datavalue.DeflatedDataValue; -import org.hisp.dhis.importexport.*; +import org.hisp.dhis.importexport.CSVConverter; +import org.hisp.dhis.importexport.ExportParams; +import org.hisp.dhis.importexport.ImportDataValue; +import org.hisp.dhis.importexport.ImportObjectService; +import org.hisp.dhis.importexport.ImportParams; import org.hisp.dhis.importexport.analysis.ImportAnalyser; import org.hisp.dhis.importexport.importer.DataValueImporter; import org.hisp.dhis.organisationunit.OrganisationUnit; -import org.hisp.dhis.period.Period; -import org.hisp.dhis.period.PeriodService; + import org.hisp.dhis.system.util.DateUtils; import org.hisp.dhis.system.util.MimicingHashMap; import org.hisp.dhis.system.util.StreamUtils; -import java.io.BufferedReader; -import java.io.IOException; -import java.util.Collection; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import static org.hisp.dhis.system.util.CsvUtils.*; - /** * @author Lars Helge Overland * @version $Id$ */ public class DataValueConverter - extends DataValueImporter implements CSVConverter + extends DataValueImporter + implements CSVConverter { + private static final String SEPARATOR = ","; + private static final String FILENAME = "RoutineData.txt"; private DataElementCategoryService categoryService; + private PeriodService periodService; + private StatementManager statementManager; + private DataElementService dataElementService; - // ------------------------------------------------------------------------- // Properties // ------------------------------------------------------------------------- - private Map dataElementMapping; - private Map periodMapping; + private Map dataElementMapping; + + private Map periodMapping; + private Map sourceMapping; // ------------------------------------------------------------------------- @@ -91,15 +107,13 @@ this.statementManager = statementManager; this.dataElementService = dataElementService; } - + /** * Constructor for read operations. */ public DataValueConverter( BatchHandler importDataValueBatchHandler, - DataElementCategoryService categoryService, - ImportObjectService importObjectService, - ImportAnalyser importAnalyser, - ImportParams params ) + DataElementCategoryService categoryService, ImportObjectService importObjectService, + ImportAnalyser importAnalyser, ImportParams params ) { this.importDataValueBatchHandler = importDataValueBatchHandler; this.categoryService = categoryService; @@ -114,13 +128,13 @@ // ------------------------------------------------------------------------- // CSVConverter implementation // ------------------------------------------------------------------------- - + public void write( ZipOutputStream out, ExportParams params ) { try { out.putNextEntry( new ZipEntry( FILENAME ) ); - + out.write( getCsvValue( csvEncode( "RoutineDataID" ) ) ); out.write( getCsvValue( csvEncode( "OrgUnitID" ) ) ); out.write( getCsvValue( csvEncode( "DataElementID" ) ) ); @@ -137,47 +151,49 @@ out.write( getCsvValue( csvEncode( "Comment" ) ) ); out.write( getCsvValue( csvEncode( "LastUserID" ) ) ); out.write( getCsvEndValue( csvEncode( "LastUpdated" ) ) ); - + out.write( NEWLINE ); - + if ( params.isIncludeDataValues() ) { if ( params.getStartDate() != null && params.getEndDate() != null ) { Collection values = null; - - Collection periods = periodService.getIntersectingPeriods( params.getStartDate(), params.getEndDate() ); - + + Collection periods = periodService.getIntersectingPeriods( params.getStartDate(), + params.getEndDate() ); + statementManager.initialise(); - + for ( final Integer element : params.getDataElements() ) { for ( final Period period : periods ) { - values = aggregatedDataValueService.getDeflatedDataValues( element, period.getId(), params.getOrganisationUnits() ); - + values = aggregatedDataValueService.getDeflatedDataValues( element, period.getId(), + params.getOrganisationUnits() ); + for ( final DeflatedDataValue value : values ) { out.write( getCsvValue( 0 ) ); out.write( getCsvValue( value.getSourceId() ) ); out.write( getCsvValue( value.getDataElementId() ) ); out.write( getCsvValue( value.getPeriodId() ) ); - out = getCSVDataExportField(out, value ); + out = getCSVDataExportField( out, value ); out.write( getCsvValue( 0 ) ); out.write( getCsvValue( 0 ) ); out.write( getCsvValue( 0 ) ); out.write( getCsvValue( csvEncode( value.getComment() ) ) ); out.write( getCsvValue( 1 ) ); out.write( getCsvEndValue( DateUtils.getAccessDateString( value.getTimestamp() ) ) ); - + out.write( NEWLINE ); } } } - + statementManager.destroy(); } - } + } StreamUtils.closeZipEntry( out ); } @@ -190,8 +206,7 @@ public void read( BufferedReader reader, ImportParams params ) { String line = ""; - Boolean validValue = true; - + DataValue value = new DataValue(); DataElement dataElement = new DataElement(); Period period = new Period(); @@ -199,61 +214,65 @@ DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); DataElementCategoryOptionCombo proxyCategoryOptionCombo = new DataElementCategoryOptionCombo(); proxyCategoryOptionCombo.setId( categoryOptionCombo.getId() ); - final String owner = params.getOwner(); + final String owner = params.getOwner(); try { reader.readLine(); // Skip CSV header - - while( ( line = reader.readLine() ) != null ) + + while ( (line = reader.readLine()) != null ) { String[] values = line.split( SEPARATOR ); + Boolean validValue = true; dataElement.setId( dataElementMapping.get( Integer.parseInt( values[2] ) ) ); period.setId( periodMapping.get( Integer.parseInt( values[3] ) ) ); organisationUnit.setId( sourceMapping.get( Integer.parseInt( values[1] ) ) ); - + value.setDataElement( dataElement ); value.setPeriod( period ); value.setSource( organisationUnit ); - if ( values[4] != null || !values[4].isEmpty() ) //Text + if ( values[4] != null || !values[4].isEmpty() ) // Text { value.setValue( values[4].trim() ); } - else if ( values[5] != null || !values[5].isEmpty() ) //Boolean + else if ( values[5] != null || !values[5].isEmpty() ) // Boolean { + value.setValue( "false" ); - if ( values[5].trim().equals( "1" ) ) + if ( values[5].trim().equals( "1" ) ) { value.setValue( "true" ); } } - else if ( values[6] != null || !values[6].isEmpty() ) //Numeric + else if ( values[6] != null || !values[6].isEmpty() ) // Numeric { value.setValue( handleNumericValue( values[6] ) ); validValue = isValidNumeric( value.getValue() ); } - else if ( values[7] != null || !values[7].isEmpty() ) //Date + else if ( values[7] != null || !values[7].isEmpty() ) // Date { try { - value.setValue( handleNumericValue( DateUtils.convertDate(values[7]) ) ); + value.setValue( handleNumericValue( DateUtils.convertDate( values[7] ) ) ); } - catch (Exception e) + catch ( Exception e ) { validValue = false; } - } - else if ( values[8] != null || !values[8].isEmpty() ) //Memo not supported + else if ( values[8] != null || !values[8].isEmpty() ) // Memo + // not + // supported { validValue = false; } - else if ( values[9] != null || !values[9].isEmpty() ) //OLE not supported + else if ( values[9] != null || !values[9].isEmpty() ) // OLE not + // supported { validValue = false; } @@ -262,7 +281,7 @@ value.setTimestamp( DateUtils.getDefaultDate( values[15] ) ); value.setOptionCombo( proxyCategoryOptionCombo ); value.setStoredBy( owner ); - + if ( validValue ) { importObject( value, params ); @@ -272,37 +291,37 @@ catch ( IOException ex ) { throw new RuntimeException( "Failed to read data", ex ); - } + } } // ------------------------------------------------------------------------- // CSVConverter implementation // ------------------------------------------------------------------------- - + private String handleNumericValue( String value ) { - if ( value != null ) + if ( value != null ) { - //Remove all spaces - value = value.replaceAll(" ", ""); - //Remove all quotes + // Remove all spaces + value = value.replaceAll( " ", "" ); + // Remove all quotes value = value.replaceAll( "\"", "" ); - //Strip trailing zeros - value = value.replaceAll("\\.0+$", ""); + // Strip trailing zeros + value = value.replaceFirst( "\\.0+$", "" ); } return value; } - private boolean isValidNumeric (String value) + private boolean isValidNumeric( String value ) { - return value != null && value.matches("-?\\d+(\\.\\d+)?"); + return value != null && value.matches( "-?\\d+(\\.\\d+)?" ); } - private ZipOutputStream getCSVDataExportField (ZipOutputStream out, DeflatedDataValue value) + private ZipOutputStream getCSVDataExportField( ZipOutputStream out, DeflatedDataValue value ) { - - String dataElementType = dataElementService.getDataElement(value.getDataElementId()).getType(); + + String dataElementType = dataElementService.getDataElement( value.getDataElementId() ).getType(); try { @@ -334,14 +353,14 @@ out.write( SEPARATOR_B ); out.write( SEPARATOR_B ); out.write( SEPARATOR_B ); - } + } - else if ( dataElementType.equals(DataElement.VALUE_TYPE_DATE ) ) + else if ( dataElementType.equals( DataElement.VALUE_TYPE_DATE ) ) { out.write( SEPARATOR_B ); out.write( SEPARATOR_B ); out.write( SEPARATOR_B ); - out.write( getCsvValue( DateUtils.getMediumDate( value.getValue() ).toString() ) ) ; + out.write( getCsvValue( DateUtils.getMediumDate( value.getValue() ).toString() ) ); out.write( SEPARATOR_B ); out.write( SEPARATOR_B ); } === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java 2012-06-25 18:21:40 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/dhis14/xml/importer/DefaultDhis14XMLImportService.java 2012-06-27 07:16:03 +0000 @@ -27,22 +27,44 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + import org.amplecode.quick.BatchHandler; import org.amplecode.quick.BatchHandlerFactory; import org.amplecode.staxwax.factory.XMLFactory; import org.amplecode.staxwax.reader.XMLReader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.hisp.dhis.cache.HibernateCacheManager; import org.hisp.dhis.common.ProcessState; import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo; import org.hisp.dhis.dataelement.DataElementCategoryService; import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.expression.ExpressionService; -import org.hisp.dhis.importexport.*; +import org.hisp.dhis.importexport.CSVConverter; +import org.hisp.dhis.importexport.ImportDataValue; +import org.hisp.dhis.importexport.ImportObjectService; +import org.hisp.dhis.importexport.ImportParams; +import org.hisp.dhis.importexport.ImportService; +import org.hisp.dhis.importexport.XMLConverter; import org.hisp.dhis.importexport.analysis.DefaultImportAnalyser; import org.hisp.dhis.importexport.analysis.ImportAnalyser; -import org.hisp.dhis.importexport.dhis14.xml.converter.*; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryComboConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryOptionComboConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementCategoryOptionConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataElementConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.DataValueConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.IndicatorTypeConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.OrganisationUnitHierarchyConverter; +import org.hisp.dhis.importexport.dhis14.xml.converter.PeriodConverter; import org.hisp.dhis.importexport.invoker.ConverterInvoker; import org.hisp.dhis.importexport.mapping.NameMappingUtil; import org.hisp.dhis.importexport.mapping.ObjectMappingGenerator; @@ -54,28 +76,23 @@ import org.hisp.dhis.system.util.StreamUtils; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - /** * @author Lars Helge Overland * @version $Id$ */ public class DefaultDhis14XMLImportService - implements ImportService +implements ImportService { private static final Log log = LogFactory.getLog( DefaultDhis14XMLImportService.class ); - + private static final String DATA_FILE_NAME = "routinedata.txt"; + private static final String META_DATA_FILE_SUFFIX = ".xml"; - + // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- - + private DataElementCategoryService categoryService; public void setCategoryService( DataElementCategoryService categoryService ) @@ -84,19 +101,19 @@ } private DataElementService dataElementService; - + public void setDataElementService( DataElementService dataElementService ) { this.dataElementService = dataElementService; } - + private IndicatorService indicatorService; public void setIndicatorService( IndicatorService indicatorService ) { this.indicatorService = indicatorService; } - + private PeriodService periodService; public void setPeriodService( PeriodService periodService ) @@ -117,21 +134,21 @@ { this.importObjectService = importObjectService; } - + private BatchHandlerFactory batchHandlerFactory; public void setBatchHandlerFactory( BatchHandlerFactory batchHandlerFactory ) { this.batchHandlerFactory = batchHandlerFactory; } - + private ObjectMappingGenerator objectMappingGenerator; public void setObjectMappingGenerator( ObjectMappingGenerator objectMappingGenerator ) { this.objectMappingGenerator = objectMappingGenerator; } - + private ConverterInvoker converterInvoker; public void setConverterInvoker( ConverterInvoker converterInvoker ) @@ -140,7 +157,7 @@ } private ExpressionService expressionService; - + public void setExpressionService( ExpressionService expressionService ) { this.expressionService = expressionService; @@ -163,7 +180,7 @@ { super(); } - + // ------------------------------------------------------------------------- // ImportService implementation // ------------------------------------------------------------------------- @@ -172,63 +189,71 @@ { importData( params, inputStream, new OutputHolderState() ); } - + public void importData( ImportParams params, InputStream inputStream, ProcessState state ) - { - DataElementCategoryOptionCombo defaultCategoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); - + { + DataElementCategoryOptionCombo defaultCategoryOptionCombo = categoryService + .getDefaultDataElementCategoryOptionCombo(); + NameMappingUtil.clearMapping(); - + importAnalyser = new DefaultImportAnalyser( expressionService ); - - if ( !( params.isPreview() || params.isAnalysis() ) ) + + if ( !(params.isPreview() || params.isAnalysis()) ) { throw new RuntimeException( "Only preview mode allowed for DHIS 1.4 XML import" ); } - + importObjectService.deleteImportObjects(); - ZipInputStream zipIn = new ZipInputStream ( inputStream ); + ZipInputStream zipIn = new ZipInputStream( inputStream ); ZipEntry zipEntry = StreamUtils.getNextZipEntry( zipIn ); - + while ( zipEntry != null ) { log.info( "Reading file: " + zipEntry.getName() ); - + if ( zipEntry.getName().toLowerCase().trim().endsWith( META_DATA_FILE_SUFFIX ) ) { // ------------------------------------------------------------- // Meta-data // ------------------------------------------------------------- - - state.setMessage( "importing_meta_data" ); + state.setMessage( "importing_meta_data" ); log.info( "Importing meta data" ); - + XMLReader reader = XMLFactory.getXMLReader( zipIn ); - - XMLConverter categoryOptionConverter = new DataElementCategoryOptionConverter( importObjectService, categoryService ); + + XMLConverter categoryOptionConverter = new DataElementCategoryOptionConverter( importObjectService, + categoryService ); XMLConverter categoryConverter = new DataElementCategoryConverter( importObjectService, categoryService ); - XMLConverter categoryComboConverter = new DataElementCategoryComboConverter( importObjectService, categoryService ); - XMLConverter categoryOptionComboConverter = new DataElementCategoryOptionComboConverter( importObjectService, categoryService ); - XMLConverter dataElementConverter = new DataElementConverter( importObjectService, dataElementService, categoryService, importAnalyser ); + XMLConverter categoryComboConverter = new DataElementCategoryComboConverter( importObjectService, + categoryService ); + XMLConverter categoryOptionComboConverter = new DataElementCategoryOptionComboConverter( + importObjectService, categoryService ); + XMLConverter dataElementConverter = new DataElementConverter( importObjectService, dataElementService, + categoryService, importAnalyser ); XMLConverter indicatorTypeConverter = new IndicatorTypeConverter( importObjectService, indicatorService ); - XMLConverter indicatorConverter = new IndicatorConverter( importObjectService, indicatorService, importAnalyser, defaultCategoryOptionCombo ); - XMLConverter organisationUnitConverter = new OrganisationUnitConverter( importObjectService, organisationUnitService, importAnalyser ); - XMLConverter hierarchyConverter = new OrganisationUnitHierarchyConverter( importObjectService, organisationUnitService ); - XMLConverter periodConverter = new PeriodConverter( importObjectService, periodService, objectMappingGenerator.getPeriodTypeMapping() ); + XMLConverter indicatorConverter = new IndicatorConverter( importObjectService, indicatorService, + importAnalyser, defaultCategoryOptionCombo ); + XMLConverter organisationUnitConverter = new OrganisationUnitConverter( importObjectService, + organisationUnitService, importAnalyser ); + XMLConverter hierarchyConverter = new OrganisationUnitHierarchyConverter( importObjectService, + organisationUnitService ); + XMLConverter periodConverter = new PeriodConverter( importObjectService, periodService, + objectMappingGenerator.getPeriodTypeMapping() ); converterInvoker.invokeRead( categoryOptionConverter, reader, params ); converterInvoker.invokeRead( categoryConverter, reader, params ); converterInvoker.invokeRead( categoryComboConverter, reader, params ); converterInvoker.invokeRead( categoryOptionComboConverter, reader, params ); - + while ( reader.next() ) { if ( reader.isStartElement( DataElementConverter.ELEMENT_NAME ) ) { - converterInvoker.invokeRead( dataElementConverter, reader, params ); + converterInvoker.invokeRead( dataElementConverter, reader, params ); } else if ( reader.isStartElement( IndicatorTypeConverter.ELEMENT_NAME ) ) { @@ -251,7 +276,7 @@ converterInvoker.invokeRead( periodConverter, reader, params ); } } - + reader.closeReader(); } else if ( zipEntry.getName().toLowerCase().trim().equals( DATA_FILE_NAME ) ) @@ -260,25 +285,24 @@ // Data // ------------------------------------------------------------- - state.setMessage( "importing_data_values" ); + state.setMessage( "importing_data_values" ); log.info( "Importing DataValues" ); - + BufferedReader streamReader = new BufferedReader( new InputStreamReader( zipIn ) ); - - BatchHandler importDataValueBatchHandler = batchHandlerFactory.createBatchHandler( ImportDataValueBatchHandler.class ).init(); - - CSVConverter dataValueConverter = new DataValueConverter( importDataValueBatchHandler, - categoryService, - importObjectService, - importAnalyser, - params ); - + + BatchHandler importDataValueBatchHandler = batchHandlerFactory.createBatchHandler( + ImportDataValueBatchHandler.class ).init(); + + CSVConverter dataValueConverter = new DataValueConverter( importDataValueBatchHandler, categoryService, + importObjectService, importAnalyser, params ); + dataValueConverter.read( streamReader, params ); - + importDataValueBatchHandler.flush(); } - zipEntry = StreamUtils.getNextZipEntry( zipIn ); // Move to next entry in archive + zipEntry = StreamUtils.getNextZipEntry( zipIn ); // Move to next + // entry in archive } if ( params.isAnalysis() ) @@ -289,7 +313,7 @@ StreamUtils.closeInputStream( zipIn ); NameMappingUtil.clearMapping(); - + cacheManager.clearCache(); } } === modified file 'dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/service/DefaultImportObjectManager.java' --- dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/service/DefaultImportObjectManager.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-services/dhis-service-importexport/src/main/java/org/hisp/dhis/importexport/service/DefaultImportObjectManager.java 2012-06-27 07:16:03 +0000 @@ -27,11 +27,6 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - import org.amplecode.quick.BatchHandler; import org.amplecode.quick.BatchHandlerFactory; import org.apache.commons.logging.Log; @@ -149,6 +144,11 @@ import org.hisp.dhis.validation.ValidationRuleService; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + /** * @author Lars Helge Overland * @version $Id$ @@ -1049,6 +1049,7 @@ { Integer importedObjects = 0; Integer failedObjects = 0; + Integer ignoredObjects = 0; BatchHandler batchHandler = batchHandlerFactory.createBatchHandler( DataValueBatchHandler.class ) .init(); @@ -1065,14 +1066,26 @@ for ( ImportDataValue importValue : importValues ) { DataValue value = importValue.getDataValue(); + try { - value.getDataElement().setId( dataElementMapping.get( value.getDataElement().getId() ) ); - value.getPeriod().setId( periodMapping.get( value.getPeriod().getId() ) ); - value.getSource().setId( sourceMapping.get( value.getSource().getId() ) ); - value.getOptionCombo().setId( categoryOptionComboMapping.get( value.getOptionCombo().getId() ) ); - importer.importObject( value, params ); - importedObjects++; + if ( dataElementMapping.containsKey( value.getDataElement().getId() ) + && periodMapping.containsKey( value.getPeriod().getId()) + && sourceMapping.containsKey( value.getSource().getId()) + && categoryOptionComboMapping.containsKey(value.getOptionCombo().getId())) + { + + value.getDataElement().setId( dataElementMapping.get( value.getDataElement().getId() ) ); + value.getPeriod().setId( periodMapping.get( value.getPeriod().getId() ) ); + value.getSource().setId( sourceMapping.get( value.getSource().getId() ) ); + value.getOptionCombo().setId( categoryOptionComboMapping.get( value.getOptionCombo().getId() ) ); + importer.importObject( value, params ); + importedObjects++; + } + else + { + ignoredObjects++; + } } catch ( Exception e ) @@ -1087,7 +1100,7 @@ importDataValueService.deleteImportDataValues(); - log.info( importReport( importedObjects, failedObjects ) ); + log.info( importReport( importedObjects, failedObjects, ignoredObjects ) ); } // ------------------------------------------------------------------------- @@ -1121,15 +1134,16 @@ importObjectStore.deleteImportObjects( type ); } - private String importReport( Integer importedObjects, Integer failedObjects ) + private String importReport( Integer importedObjects, Integer failedObjects, Integer ignoredObjects ) { - Integer totalObjects = importedObjects + failedObjects; + Integer totalObjects = importedObjects + failedObjects + ignoredObjects; String importReportString = ""; - if ( failedObjects > 0 ) + if ( failedObjects > 0 || ignoredObjects > 0 ) { - importReportString = totalObjects.toString() + " values handled.\n" + importedObjects.toString() - + " new values successfully imported.\n" + failedObjects.toString() - + " were not imported due to errors."; + importReportString = totalObjects.toString() + " values handled.\n" + + importedObjects.toString() + " new values successfully imported.\n" + + ignoredObjects.toString() + " were ignored due to validitiy errors.\n" + + failedObjects.toString()+ " were not imported due to other errors."; return importReportString; } else