=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java 2015-11-02 17:18:46 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/AdxDataService.java 2015-11-02 18:56:53 +0000 @@ -71,7 +71,7 @@ * * @param in the InputStream. * @param importOptions the importOptions. - * @param id the task id. + * @param id the task id, can be null. * * @return an ImportSummaries collection of ImportSummary for each DataValueSet. */ === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-11-02 17:18:46 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultAdxDataService.java 2015-11-02 18:56:53 +0000 @@ -80,6 +80,7 @@ import org.hisp.dhis.scheduling.TaskId; import org.hisp.dhis.system.notification.Notifier; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; /** * @author bobj @@ -128,6 +129,7 @@ } @Override + @Transactional public ImportSummaries saveDataValueSet( InputStream in, ImportOptions importOptions, TaskId id ) { notifier.clear( id ).notify( id, "ADX parsing process started" ); @@ -148,7 +150,7 @@ try ( PipedOutputStream pipeOut = new PipedOutputStream() ) { Future futureImportSummary; - futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, pipeOut ) ); + futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, id, pipeOut ) ); XMLOutputFactory factory = XMLOutputFactory.newInstance(); XMLStreamWriter dxfWriter = factory.createXMLStreamWriter( pipeOut ); @@ -165,13 +167,14 @@ summary.getConflicts().add( conflict ); summary.getImportCount().incrementIgnored(); } + importSummaries.addImportSummary( summary ); } catch ( AdxException ex) { ImportSummary importSummary = new ImportSummary(); importSummary.setStatus( ImportStatus.ERROR ); - importSummary.setDescription( "DataSet import failed for group number " + count ); + importSummary.setDescription( "Data set import failed for group number: " + count ); importSummary.getConflicts().add( ex.getImportConflict() ); importSummaries.addImportSummary( importSummary ); log.warn( "Import failed: " + ex ); @@ -180,7 +183,7 @@ { ImportSummary importSummary = new ImportSummary(); importSummary.setStatus( ImportStatus.ERROR ); - importSummary.setDescription( "DataSet import failed for group number " + count ); + importSummary.setDescription( "Data set import failed for group number: " + count ); importSummaries.addImportSummary( importSummary ); log.warn( "Import failed: " + ex ); } @@ -190,8 +193,6 @@ executor.shutdown(); - notifier.notify( id, "ADX parsing done" ); - return importSummaries; } @@ -222,7 +223,7 @@ throw new AdxException( AdxDataService.ORGUNIT + " attribute is required on 'group'" ); } - // translate ADX period to dxf2 + // translate ADX period to DXF String periodStr = groupAttributes.get( AdxDataService.PERIOD ); groupAttributes.remove( AdxDataService.PERIOD ); Period period = AdxPeriod.parse( periodStr ); @@ -232,14 +233,14 @@ if ( !groupAttributes.containsKey( AdxDataService.ATTOPTCOMBO ) && groupAttributes.containsKey( AdxDataService.DATASET ) ) { - log.debug( "No attributeOptionCombo present. Check dataSet for attribute categorycombo" ); + log.debug( "No attribute option combo present, check data set for attribute category combo" ); DataSet dataSet = identifiableObjectManager.getObject( DataSet.class, dataElementIdScheme, groupAttributes.get( AdxDataService.DATASET ) ); - if (dataSet == null) + if ( dataSet == null ) { - throw new AdxException("No dataSet matching identifier: " + groupAttributes.get( AdxDataService.DATASET )); + throw new AdxException("No data set matching identifier: " + groupAttributes.get( AdxDataService.DATASET ) ); } groupAttributes.put( AdxDataService.DATASET, dataSet.getUid() ); @@ -263,7 +264,8 @@ catch (AdxException ex) { adxConflicts.add( ex.getImportConflict() ); - log.info("ADX datavalue conflict: " + ex.getImportConflict()); + + log.info("ADX data value conflict: " + ex.getImportConflict() ); } } @@ -278,7 +280,7 @@ { Map dvAttributes = readAttributes( adxReader ); - log.debug("Processing datavalue: " + dvAttributes ); + log.debug( "Processing data value: " + dvAttributes ); if ( !dvAttributes.containsKey( AdxDataService.DATAELEMENT ) ) { @@ -292,17 +294,18 @@ IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme(); - DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme,dvAttributes.get( AdxDataService.DATAELEMENT)); + DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme, dvAttributes.get( AdxDataService.DATAELEMENT ) ); if ( dataElement == null ) { - throw new AdxException(dvAttributes.get( AdxDataService.DATAELEMENT), "No matching dataelement"); + throw new AdxException(dvAttributes.get( AdxDataService.DATAELEMENT), "No matching dataelement" ); } // process ADX data value attributes if ( !dvAttributes.containsKey( AdxDataService.CATOPTCOMBO ) ) { - log.debug( "No categoryOptionCombo present." ); + log.debug( "No category option combo present" ); + DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo(); attributesToDxf( AdxDataService.CATOPTCOMBO, categoryCombo, dvAttributes, dataElementIdScheme ); @@ -312,6 +315,7 @@ if ( dataElement.getValueType().isText() ) { adxReader.moveToStartElement( AdxDataService.ANNOTATION, AdxDataService.DATAVALUE ); + if ( adxReader.isStartElement( AdxDataService.ANNOTATION ) ) { String textValue = adxReader.getElementValue(); @@ -323,7 +327,7 @@ } } - log.debug("Processing datavalue as DXF2: " + dvAttributes ); + log.debug( "Processing data value as DXF: " + dvAttributes ); dxfWriter.writeStartElement( "dataValue" ); @@ -332,15 +336,16 @@ { dxfWriter.writeAttribute( attribute, dvAttributes.get( attribute ) ); } + dxfWriter.writeEndElement(); } - private Map createCategoryMap( DataElementCategoryCombo catcombo ) + private Map createCategoryMap( DataElementCategoryCombo categoryCombo ) throws AdxException { Map categoryMap = new HashMap<>(); - List categories = catcombo.getCategories(); + List categories = categoryCombo.getCategories(); for ( DataElementCategory category : categories ) { @@ -380,38 +385,38 @@ for ( DataElementCategory category : catcomboMap.getCategories() ) { - String catCode = category.getCode(); + String categoryCode = category.getCode(); - if ( catCode == null ) + if ( categoryCode == null ) { - throw new AdxException( "No category matching " + catCode ); + throw new AdxException( "No category matching: " + categoryCode ); } - String catAttribute = attributes.get( catCode ); + String catAttribute = attributes.get( categoryCode ); if ( catAttribute == null ) { - throw new AdxException( "Missing required attribute from catcombo: " + catCode ); + throw new AdxException( "Missing required attribute from catcombo: " + categoryCode ); } compositeIdentifier += "\"" + catAttribute + "\""; } - DataElementCategoryOptionCombo catoptcombo = catcomboMap.getCategoryOptionCombo( compositeIdentifier ); + DataElementCategoryOptionCombo catOptionCombo = catcomboMap.getCategoryOptionCombo( compositeIdentifier ); - if ( catoptcombo == null ) + if ( catOptionCombo == null ) { throw new AdxException( "Invalid attributes:" + attributes ); } - return catoptcombo; + return catOptionCombo; } private void attributesToDxf( String optionComboName, DataElementCategoryCombo catCombo, Map attributes, IdentifiableProperty scheme ) throws AdxException { - log.debug( "adx attributes: " + attributes ); + log.debug( "ADX attributes: " + attributes ); if ( catCombo.isDefault() ) { @@ -431,14 +436,15 @@ } else { - throw new AdxException( "catcombo " + catCombo.getName() + " must have " + categoryMap.get( category ).getName() ); + throw new AdxException( "Category combo " + catCombo.getName() + " must have " + categoryMap.get( category ).getName() ); } } DataElementCategoryOptionCombo catOptCombo = getCatOptComboFromAttributes( attributeOptions, catCombo, scheme ); + attributes.put( optionComboName, catOptCombo.getUid() ); - log.debug( "dxf attributes: " + attributes ); + log.debug( "DXF attributes: " + attributes ); } // TODO this should be part of staxwax library === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java 2015-10-05 17:45:17 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/PipedImporter.java 2015-11-02 18:56:53 +0000 @@ -37,6 +37,7 @@ import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.importsummary.ImportStatus; import org.hisp.dhis.dxf2.importsummary.ImportSummary; +import org.hisp.dhis.scheduling.TaskId; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; @@ -56,14 +57,17 @@ private final ImportOptions importOptions; + private final TaskId id; + private final Authentication authentication; public PipedImporter( DataValueSetService dataValueSetService, ImportOptions importOptions, - PipedOutputStream pipeOut ) throws IOException + TaskId id, PipedOutputStream pipeOut ) throws IOException { this.dataValueSetService = dataValueSetService; this.pipeIn = new PipedInputStream( pipeOut, PIPE_BUFFER_SIZE ); this.importOptions = importOptions; + this.id = id; this.authentication = SecurityContextHolder.getContext().getAuthentication(); } @@ -75,7 +79,7 @@ try { - result = dataValueSetService.saveDataValueSet( pipeIn, importOptions ); + result = dataValueSetService.saveDataValueSet( pipeIn, importOptions, id ); } catch ( Exception ex ) { === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java 2015-10-31 03:58:41 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/ImportDataValueTask.java 2015-11-02 18:56:53 +0000 @@ -29,6 +29,7 @@ */ import org.hisp.dhis.security.SecurityContextRunnable; +import org.hisp.dhis.dxf2.adx.AdxDataService; import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.datavalueset.DataValueSetService; import org.hisp.dhis.scheduling.TaskId; @@ -45,22 +46,27 @@ public static final String FORMAT_JSON = "json"; public static final String FORMAT_CSV = "csv"; public static final String FORMAT_PDF = "pdf"; + public static final String FORMAT_ADX = "adx"; private DataValueSetService dataValueSetService; + private AdxDataService adxDataService; + private InputStream inputStream; - private final ImportOptions options; + private final ImportOptions importOptions; private final TaskId taskId; private final String format; - public ImportDataValueTask( DataValueSetService dataValueSetService, InputStream inputStream, ImportOptions options, TaskId taskId, String format ) + public ImportDataValueTask( DataValueSetService dataValueSetService, AdxDataService adxDataService, + InputStream inputStream, ImportOptions importOptions, TaskId taskId, String format ) { this.dataValueSetService = dataValueSetService; + this.adxDataService = adxDataService; this.inputStream = inputStream; - this.options = options; + this.importOptions = importOptions; this.taskId = taskId; this.format = format; } @@ -70,19 +76,23 @@ { if ( FORMAT_JSON.equals( format ) ) { - dataValueSetService.saveDataValueSetJson( inputStream, options, taskId ); + dataValueSetService.saveDataValueSetJson( inputStream, importOptions, taskId ); } else if ( FORMAT_CSV.equals( format ) ) { - dataValueSetService.saveDataValueSetCsv( inputStream, options, taskId ); + dataValueSetService.saveDataValueSetCsv( inputStream, importOptions, taskId ); } else if ( FORMAT_PDF.equals( format ) ) { - dataValueSetService.saveDataValueSetPdf( inputStream, options, taskId ); + dataValueSetService.saveDataValueSetPdf( inputStream, importOptions, taskId ); + } + else if ( FORMAT_ADX.equals( format ) ) + { + adxDataService.saveDataValueSet( inputStream, importOptions, taskId ); } else // FORMAT_XML { - dataValueSetService.saveDataValueSet( inputStream, options, taskId ); + dataValueSetService.saveDataValueSet( inputStream, importOptions, taskId ); } } } === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java 2015-11-02 05:14:38 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueSetController.java 2015-11-02 18:56:53 +0000 @@ -1,5 +1,7 @@ package org.hisp.dhis.webapi.controller; +import org.hisp.dhis.dxf2.adx.AdxDataService; + /* * Copyright (c) 2004-2015, University of Oslo * All rights reserved. @@ -33,6 +35,7 @@ import org.hisp.dhis.dxf2.datavalueset.DataExportParams; import org.hisp.dhis.dxf2.datavalueset.DataValueSetService; import org.hisp.dhis.dxf2.datavalueset.tasks.ImportDataValueTask; +import org.hisp.dhis.dxf2.importsummary.ImportSummaries; import org.hisp.dhis.dxf2.importsummary.ImportSummary; import org.hisp.dhis.dxf2.render.RenderService; import org.hisp.dhis.scheduling.TaskCategory; @@ -76,6 +79,9 @@ private DataValueSetService dataValueSetService; @Autowired + private AdxDataService adxDataService; + + @Autowired private RenderService renderService; @Autowired @@ -171,6 +177,24 @@ } } + @RequestMapping( method = RequestMethod.POST, consumes = "application/xml+adx" ) + @PreAuthorize( "hasRole('ALL') or hasRole('F_DATAVALUE_ADD')" ) + public void postAdxDataValueSet( ImportOptions importOptions, + HttpServletRequest request, HttpServletResponse response ) throws IOException + { + if ( importOptions.isAsync() ) + { + startAsyncImport( importOptions, ImportDataValueTask.FORMAT_ADX, request, response ); + } + else + { + ImportSummaries summaries = adxDataService.saveDataValueSet( request.getInputStream(), importOptions, null ); + + response.setContentType( CONTENT_TYPE_XML ); + renderService.toXml( response.getOutputStream(), summaries ); + } + } + @RequestMapping( method = RequestMethod.POST, consumes = "application/json" ) @PreAuthorize( "hasRole('ALL') or hasRole('F_DATAVALUE_ADD')" ) public void postJsonDataValueSet( ImportOptions importOptions, @@ -226,7 +250,7 @@ InputStream inputStream = saveTmp( request.getInputStream() ); TaskId taskId = new TaskId( TaskCategory.DATAVALUE_IMPORT, currentUserService.getCurrentUser() ); - scheduler.executeTask( new ImportDataValueTask( dataValueSetService, inputStream, importOptions, taskId, format ) ); + scheduler.executeTask( new ImportDataValueTask( dataValueSetService, adxDataService, inputStream, importOptions, taskId, format ) ); response.setHeader( "Location", ContextUtils.getRootPath( request ) + "/system/tasks/" + TaskCategory.DATAVALUE_IMPORT ); response.setStatus( HttpServletResponse.SC_ACCEPTED ); === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-10-31 03:58:41 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/datavalue/ImportDataValueAction.java 2015-11-02 18:56:53 +0000 @@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.IdentifiableProperty; import org.hisp.dhis.commons.util.StreamUtils; +import org.hisp.dhis.dxf2.adx.AdxDataService; import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.datavalueset.DataValueSetService; import org.hisp.dhis.dxf2.datavalueset.tasks.ImportDataValueTask; @@ -59,6 +60,9 @@ @Autowired private DataValueSetService dataValueSetService; + + @Autowired + private AdxDataService adxDataService; @Autowired private CurrentUserService currentUserService; @@ -160,7 +164,7 @@ log.info( options ); - scheduler.executeTask( new ImportDataValueTask( dataValueSetService, in, options, taskId, importFormat ) ); + scheduler.executeTask( new ImportDataValueTask( dataValueSetService, adxDataService, in, options, taskId, importFormat ) ); return SUCCESS; } === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm' --- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2015-08-30 19:03:15 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importDataValue.vm 2015-11-02 18:56:53 +0000 @@ -18,6 +18,7 @@ +