=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java 2015-07-03 16:31:04 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXException.java 2015-07-08 16:19:15 +0000 @@ -26,16 +26,38 @@ * POSSIBILITY OF SUCH DAMAGE. */ +import org.hisp.dhis.dxf2.importsummary.ImportConflict; + /** - * A simple wrapper for ADX checked exceptions + * A simple class for ADX checked exceptions + * which can wrap an ImportConflict * * @author bobj */ public class ADXException extends Exception { + protected String object; + + public String getObject() + { + return object; + } + public ADXException(String msg) { super(msg); } + + // for wrapping ImportConflict + public ADXException(String object, String msg) + { + super(msg); + this.object = object; + } + + public ImportConflict getImportConflict() + { + return new ImportConflict(object, this.getMessage()); + } } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java 2015-06-16 18:30:37 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/ADXPeriod.java 2015-07-08 16:19:15 +0000 @@ -56,14 +56,6 @@ */ public class ADXPeriod { - public static class ADXPeriodException extends RuntimeException - { - - ADXPeriodException( String message ) - { - super( message ); - } - } public static enum Duration { @@ -78,13 +70,13 @@ private static SimpleDateFormat dateFormat = new SimpleDateFormat( "yyyy-MM-dd" ); - public static Period parse( String periodString ) throws ADXPeriodException + public static Period parse( String periodString ) throws ADXException { String[] tokens = periodString.split( "/" ); if ( tokens.length != 2 ) { - throw new ADXPeriodException( periodString + " not in valid / format" ); + throw new ADXException( periodString + " not in valid / format" ); } try @@ -123,7 +115,7 @@ periodType = new SixMonthlyAprilPeriodType(); break; default: - throw new ADXPeriodException( periodString + "is invalid sixmonthly type" ); + throw new ADXException( periodString + "is invalid sixmonthly type" ); } case P1Y: switch ( cal.get( Calendar.MONTH ) ) @@ -141,7 +133,7 @@ periodType = new FinancialOctoberPeriodType(); break; default: - throw new ADXPeriodException( periodString + "is invalid yearly type" ); + throw new ADXException( periodString + "is invalid yearly type" ); } } @@ -151,7 +143,7 @@ } else { - throw new ADXPeriodException( "Failed to create period type from " + duration ); + throw new ADXException( "Failed to create period type from " + duration ); } return period; @@ -159,11 +151,11 @@ } catch ( ParseException ex ) { - throw new ADXPeriodException( tokens[0] + "is not a valid date in YYYY-MM-dd format" ); + throw new ADXException( tokens[0] + "is not a valid date in YYYY-MM-dd format" ); } catch ( IllegalArgumentException ex ) { - throw new ADXPeriodException( tokens[1] + " is not a supported duration type" ); + throw new ADXException( tokens[1] + " is not a supported duration type" ); } } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java 2015-07-06 08:54:27 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/adx/DefaultADXDataService.java 2015-07-08 16:19:15 +0000 @@ -36,6 +36,7 @@ import java.io.PipedOutputStream; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,7 +69,6 @@ import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.dataset.DataSet; import org.hisp.dhis.dataset.DataSetService; -import org.hisp.dhis.dxf2.adx.ADXPeriod.ADXPeriodException; import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.datavalueset.DataExportParams; import org.hisp.dhis.dxf2.datavalueset.DataValueSetService; @@ -79,6 +79,7 @@ import org.hisp.dhis.period.Period; import org.springframework.beans.factory.annotation.Autowired; import org.apache.xerces.util.XMLChar; +import org.hisp.dhis.dxf2.importsummary.ImportConflict; /** * @author bobj @@ -140,6 +141,7 @@ ExecutorService executor = Executors.newSingleThreadExecutor(); + int count = 0; // submit each ADX group to DXF importer as a datavalueSet while ( adxReader.moveToStartElement( ADXConstants.GROUP, ADXConstants.NAMESPACE ) ) { @@ -149,20 +151,40 @@ futureImportSummary = executor.submit( new PipedImporter( dataValueSetService, importOptions, pipeOut ) ); XMLOutputFactory factory = XMLOutputFactory.newInstance(); XMLStreamWriter dxfWriter = factory.createXMLStreamWriter( pipeOut ); - parseADXGroupToDxf( adxReader, dxfWriter, importOptions ); + + // note theis returns conflicts which are detected at adx level + List adxConflicts = parseADXGroupToDxf( adxReader, dxfWriter, importOptions ); + pipeOut.flush(); - - importSummaries.addImportSummary( futureImportSummary.get( TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES ) ); - } - catch ( IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException - | ADXException | ADXPeriodException ex ) - { - ImportSummary importSummary = new ImportSummary(); - importSummary.setStatus( ImportStatus.ERROR ); - importSummary.setDescription( "Exception: " + ex.getMessage() ); - importSummaries.addImportSummary( importSummary ); - log.warn( "Import failed: " + ex ); - } + + ImportSummary summary = futureImportSummary.get( TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES ); + + // add adx conflicts to the import summary + for ( ImportConflict conflict : adxConflicts) + { + summary.getConflicts().add( conflict ); + summary.getImportCount().incrementIgnored(); + } + importSummaries.addImportSummary( summary ); + } + catch ( ADXException ex) + { + ImportSummary importSummary = new ImportSummary(); + importSummary.setStatus( ImportStatus.ERROR ); + importSummary.setDescription( "DataSet import failed for group number " + count ); + importSummary.getConflicts().add( ex.getImportConflict() ); + importSummaries.addImportSummary( importSummary ); + log.warn( "Import failed: " + ex ); + } + catch ( IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex ) + { + ImportSummary importSummary = new ImportSummary(); + importSummary.setStatus( ImportStatus.ERROR ); + importSummary.setDescription( "DataSet import failed for group number " + count ); + importSummaries.addImportSummary( importSummary ); + log.warn( "Import failed: " + ex ); + } + count++; } executor.shutdown(); @@ -174,9 +196,11 @@ // Utility methods // ------------------------------------------------------------------------- - private void parseADXGroupToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions ) - throws XMLStreamException, ADXException, ADXPeriodException + private List parseADXGroupToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions ) + throws XMLStreamException, ADXException { + List adxConflicts = new LinkedList<>(); + dxfWriter.writeStartDocument( "1.0" ); dxfWriter.writeStartElement( "dataValueSet" ); dxfWriter.writeDefaultNamespace( "http://dhis2.org/schema/dxf/2.0" ); @@ -199,7 +223,7 @@ String periodStr = groupAttributes.get( ADXConstants.PERIOD ); groupAttributes.remove( ADXConstants.PERIOD ); Period period = ADXPeriod.parse( periodStr ); - dxfWriter.writeAttribute( "period", period.getIsoDate() ); + groupAttributes.put( ADXConstants.PERIOD, period.getIsoDate()); // process adx group attributes if ( !groupAttributes.containsKey( ADXConstants.ATTOPTCOMBO ) @@ -209,6 +233,11 @@ DataSet dataSet = identifiableObjectManager.getObject( DataSet.class, dataElementIdScheme, groupAttributes.get( ADXConstants.DATASET ) ); + + if (dataSet == null) + { + throw new ADXException("No dataSet matching identifier: " + groupAttributes.get( ADXConstants.DATASET )); + } groupAttributes.put( ADXConstants.DATASET, dataSet.getUid() ); DataElementCategoryCombo attributeCombo = dataSet.getCategoryCombo(); attributesToDXF( ADXConstants.ATTOPTCOMBO, attributeCombo, groupAttributes, dataElementIdScheme ); @@ -223,18 +252,30 @@ // process the dataValues while ( adxReader.moveToStartElement( ADXConstants.DATAVALUE, ADXConstants.GROUP ) ) { - parseADXDataValueToDxf( adxReader, dxfWriter, importOptions ); + try + { + parseADXDataValueToDxf( adxReader, dxfWriter, importOptions ); + } + catch (ADXException ex) + { + adxConflicts.add( ex.getImportConflict() ); + log.info("ADX datavalue conflict: " + ex.getImportConflict()); + } } dxfWriter.writeEndElement(); dxfWriter.writeEndDocument(); + + return adxConflicts; } private void parseADXDataValueToDxf( XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions ) throws XMLStreamException, ADXException { Map dvAttributes = readAttributes( adxReader ); - + + log.debug("Processing datavalue: " + dvAttributes ); + if ( !dvAttributes.containsKey( ADXConstants.DATAELEMENT ) ) { throw new ADXException( ADXConstants.DATAELEMENT + " attribute is required on 'dataValue'" ); @@ -247,15 +288,12 @@ IdentifiableProperty dataElementIdScheme = importOptions.getDataElementIdScheme(); - dxfWriter.writeStartElement( "dataValue" ); - DataElement dataElement = identifiableObjectManager.getObject( DataElement.class, dataElementIdScheme,dvAttributes.get( ADXConstants.DATAELEMENT)); - // process adx group attributes - if ( !dvAttributes.containsKey( ADXConstants.CATOPTCOMBO ) - && dvAttributes.containsKey( ADXConstants.DATASET ) ) + // process adx datavalue attributes + if ( !dvAttributes.containsKey( ADXConstants.CATOPTCOMBO ) ) { - log.debug( "No attributeOptionCombo present. Check dataSet for attribute categorycombo" ); + log.debug( "No categoryOptionCombo present." ); DataElementCategoryCombo categoryCombo = dataElement.getCategoryCombo(); attributesToDXF( ADXConstants.CATOPTCOMBO, categoryCombo, dvAttributes, dataElementIdScheme ); @@ -275,12 +313,14 @@ } } + log.debug("Processing datavalue as DXF2: " + dvAttributes ); + + dxfWriter.writeStartElement( "dataValue" ); // pass through the remaining attributes to dxf for ( String attribute : dvAttributes.keySet() ) { dxfWriter.writeAttribute( attribute, dvAttributes.get( attribute ) ); } - dxfWriter.writeEndElement(); } @@ -321,7 +361,7 @@ } catch ( CategoryComboMapException ex ) { - log.warn( "Failed to create catcomboMap from " + catcombo ); + log.info( "Failed to create catcomboMap from " + catcombo ); throw new ADXException( ex.getMessage() ); } @@ -333,14 +373,14 @@ if ( catCode == null ) { - throw new RuntimeException( "No category matching " + catCode ); + throw new ADXException( "No category matching " + catCode ); } String catAttribute = attributes.get( catCode ); if ( catAttribute == null ) { - throw new RuntimeException( "Missing required attribute from catcombo: " + catCode ); + throw new ADXException( "Missing required attribute from catcombo: " + catCode ); } compositeIdentifier += "\"" + catAttribute + "\""; @@ -387,7 +427,7 @@ DataElementCategoryOptionCombo catOptCombo = getCatOptComboFromAttributes( attributeOptions, catCombo, scheme ); attributes.put( optionComboName, catOptCombo.getUid() ); - + log.debug( "dxf attributes: " + attributes ); } === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java 2015-06-11 21:37:27 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/adx/ADXPeriodTest.java 2015-07-08 16:19:15 +0000 @@ -49,20 +49,26 @@ @Test public void testParser() { - p = ADXPeriod.parse( "2015-01-01/P1Y" ); - assertEquals( "2015", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-01-01/P1M" ); - assertEquals( "201501", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-01-01/P1D" ); - assertEquals( "20150101", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-01-01/P1Q" ); - assertEquals( "2015Q1", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-04-01/P1Q" ); - assertEquals( "2015Q2", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-01-01/P7D" ); - assertEquals( "2015W1", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-01-05/P7D" ); - assertEquals( "2015W2", p.getIsoDate() ); + try { + p = ADXPeriod.parse( "2015-01-01/P1Y" ); + assertEquals( "2015", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-01-01/P1M" ); + assertEquals( "201501", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-01-01/P1D" ); + assertEquals( "20150101", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-01-01/P1Q" ); + assertEquals( "2015Q1", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-04-01/P1Q" ); + assertEquals( "2015Q2", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-01-01/P7D" ); + assertEquals( "2015W1", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-01-05/P7D" ); + assertEquals( "2015W2", p.getIsoDate() ); + } + catch (ADXException ex) + { + fail(ex.getMessage()); + } } @Test @@ -75,21 +81,28 @@ } catch ( Exception ex ) { - assertEquals( ADXPeriod.ADXPeriodException.class, ex.getClass() ); + assertEquals( ADXException.class, ex.getClass() ); } } @Test public void testFinancialTypes() { - p = ADXPeriod.parse( "2015-01-01/P1Y" ); - assertEquals( "2015", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-04-01/P1Y" ); - assertEquals( "2015April", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-07-01/P1Y" ); - assertEquals( "2015July", p.getIsoDate() ); - p = ADXPeriod.parse( "2015-10-01/P1Y" ); - assertEquals( "2015Oct", p.getIsoDate() ); + try + { + p = ADXPeriod.parse( "2015-01-01/P1Y" ); + assertEquals( "2015", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-04-01/P1Y" ); + assertEquals( "2015April", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-07-01/P1Y" ); + assertEquals( "2015July", p.getIsoDate() ); + p = ADXPeriod.parse( "2015-10-01/P1Y" ); + assertEquals( "2015Oct", p.getIsoDate() ); + } + catch (ADXException ex) + { + fail(ex.getMessage()); + } } @Test @@ -102,7 +115,7 @@ } catch ( Exception ex ) { - assertEquals( ADXPeriod.ADXPeriodException.class, ex.getClass() ); + assertEquals( ADXException.class, ex.getClass() ); } }