=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-06-03 15:00:19 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/DefaultGmlImportService.java 2015-06-05 12:56:34 +0000 @@ -32,6 +32,8 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; +import org.apache.commons.io.IOUtils; +import org.hibernate.Hibernate; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IdentifiableProperty; import org.hisp.dhis.common.MergeStrategy; @@ -88,13 +90,91 @@ // GmlImportService implementation // ------------------------------------------------------------------------- + @Transactional + @Override + public GmlPreProcessingResult preProcessGml( InputStream inputStream ) + { + InputStream dxfStream = null; + MetaData metaData = null; + + try + { + dxfStream = transformGml( inputStream ); + metaData = renderService.fromXml( dxfStream, MetaData.class ); + } + catch ( IOException | TransformerException e ) + { + return GmlPreProcessingResult.failure( e ); + } + finally + { + IOUtils.closeQuietly( dxfStream ); + } + + Map uidMap = Maps.newHashMap(), codeMap = Maps.newHashMap(), nameMap = Maps.newHashMap(); + + matchAndFilterOnIdentifiers( metaData.getOrganisationUnits(), uidMap, codeMap, nameMap ); + + Map persistedUidMap = getMatchingPersistedOrgUnits( uidMap.keySet(), IdentifiableProperty.UID ); + Map persistedCodeMap = getMatchingPersistedOrgUnits( codeMap.keySet(), IdentifiableProperty.CODE ); + Map persistedNameMap = getMatchingPersistedOrgUnits( nameMap.keySet(), IdentifiableProperty.NAME ); + + Iterator persistedIterator = Iterators.concat( persistedUidMap.values().iterator(), + persistedCodeMap.values().iterator(), persistedNameMap.values().iterator() ); + + while ( persistedIterator.hasNext() ) + { + OrganisationUnit persisted = persistedIterator.next(), imported = null; + + if ( !Strings.isNullOrEmpty( persisted.getUid() ) && uidMap.containsKey( persisted.getUid() ) ) + { + imported = uidMap.get( persisted.getUid() ); + } + else if ( !Strings.isNullOrEmpty( persisted.getCode() ) && codeMap.containsKey( persisted.getCode() ) ) + { + imported = codeMap.get( persisted.getCode() ); + } + else if ( !Strings.isNullOrEmpty( persisted.getName() ) && nameMap.containsKey( persisted.getName() ) ) + { + imported = nameMap.get( persisted.getName() ); + } + + if ( imported == null || imported.getCoordinates() == null || imported.getFeatureType() == null ) + { + continue; // Failed to dereference a persisted entity for this org unit or geo data incomplete/missing, therefore ignore + } + + mergeNonGeoData( persisted, imported ); + } + + return GmlPreProcessingResult.success( metaData ); + } + @Override public MetaData fromGml( InputStream inputStream ) throws IOException, TransformerException { - InputStream dxfStream = transformGml( inputStream ); - MetaData metaData = renderService.fromXml( dxfStream, MetaData.class ); - dxfStream.close(); + InputStream dxfStream; + MetaData metaData; + + try + { + dxfStream = transformGml( inputStream ); + } + catch (Exception e) + { + dxfStream = null; + } + + if(dxfStream != null) + { + metaData = renderService.fromXml( dxfStream, MetaData.class ); + dxfStream.close(); + } + else + { + return null; + } Map uidMap = Maps.newHashMap(), codeMap = Maps.newHashMap(), nameMap = Maps.newHashMap(); @@ -143,6 +223,7 @@ importService.importMetaData( userUid, fromGml( inputStream ), importOptions, taskId ); } + @Transactional @Override public void importGml( MetaData metaData, String userUid, ImportOptions importOptions, TaskId taskId ) { === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java 2015-06-03 15:00:19 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlImportService.java 2015-06-05 12:56:34 +0000 @@ -62,6 +62,24 @@ throws IOException, TransformerException; /** + * Pre-process a GML document. The process, in short, entails the following: + *
    + *
  1. Parse the GML payload and transform it into DXF2 format
    2. + *
  2. Get the given identifiers (uid, code or name) from the parsed payload and fetch + * the corresponding entities from the DB
    3. + *
  3. Merge the geospatial data given in the input GML into DB entities and return
    4. + *
+ * + * The result of this process in returned in a {@link GmlPreProcessingResult} which + * encapsulates the returned {@link MetaData} object or the exception in cause of parse + * failure due to IO errors or malformed input. + * + * @param gmlInputStream the InputStream providing the GML input. + * @return a GmlPreProcessingResult representing the end result of the process. + */ + GmlPreProcessingResult preProcessGml( InputStream gmlInputStream ); + + /** * Imports GML data and merges the geospatial data updates into the database. * See {@link #fromGml(InputStream)} for details on the underlying process. * === added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/gml/GmlPreProcessingResult.java 2015-06-05 12:56:34 +0000 @@ -0,0 +1,70 @@ +package org.hisp.dhis.dxf2.gml; + +import org.hisp.dhis.dxf2.metadata.MetaData; + +import java.io.InputStream; + +/** + * Wraps the result of {@link GmlImportService#preProcessGml(InputStream)}. + * This is necessary when performing GML import on a context where exceptions + * due to malformed input cannot be caught, thus leaving the user uninformed of + * the error. This class will wrap the failure and provide the Throwable to the + * consuming class on error or the resulting MetaData object on success. + * + * @author Halvdan Hoem grelland + */ +public final class GmlPreProcessingResult +{ + private boolean isSuccess; + private MetaData resultMetaData; + private Throwable throwable; + + private GmlPreProcessingResult(){} + + public static GmlPreProcessingResult success( MetaData resultMetaData ) { + GmlPreProcessingResult result = new GmlPreProcessingResult(); + result.setResultMetaData( resultMetaData ); + result.setSuccess( true ); + + return result; + } + + public static GmlPreProcessingResult failure( Throwable throwable ) + { + GmlPreProcessingResult result = new GmlPreProcessingResult(); + result.setThrowable( throwable ); + result.setSuccess( false ); + + return result; + } + + private void setSuccess( boolean isSuccess ) + { + this.isSuccess = isSuccess; + } + + public boolean isSuccess() + { + return isSuccess; + } + + private void setResultMetaData( MetaData resultMetaData ) + { + this.resultMetaData = resultMetaData; + } + + public MetaData getResultMetaData() + { + return resultMetaData; + } + + private void setThrowable( Throwable throwable ) + { + this.throwable = throwable; + } + + public Throwable getThrowable() + { + return throwable; + } +} === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java 2015-01-17 07:41:26 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/gml/GmlImportServiceTest.java 2015-06-05 12:56:34 +0000 @@ -117,4 +117,6 @@ assertEquals( 1, units.get( "Blindern").getCoordinatesAsList().get( 0 ).getNumberOfCoordinates() ); assertEquals( 76, units.get( "Forskningsparken" ).getCoordinatesAsList().get(0).getNumberOfCoordinates() ); } + + // TODO Add test for GmlImportService#preProcessGml(InputStream) } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2015-05-30 13:36:07 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/SpringScheduler.java 2015-06-05 12:56:34 +0000 @@ -77,7 +77,7 @@ { taskExecutor.execute( task ); } - + @Override public boolean scheduleTask( String key, Runnable task, String cronExpr ) { === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2015-05-28 16:10:07 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/dxf2/MetaDataImportAction.java 2015-06-05 12:56:34 +0000 @@ -187,7 +187,7 @@ } else if ( "gml".equals( importFormat ) ) { - scheduler.executeTask( new ImportMetaDataGmlTask( userId, gmlImportService, importOptions, in, taskId ) ); + scheduler.executeTask( new ImportMetaDataGmlTask( userId, gmlImportService, notifier, importOptions, in, taskId ) ); } else if ( "json".equals( importFormat ) || "xml".equals( importFormat ) ) { === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java' --- dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-04-11 14:06:51 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/java/org/hisp/dhis/importexport/action/util/ImportMetaDataGmlTask.java 2015-06-05 12:56:34 +0000 @@ -28,14 +28,18 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.dxf2.common.ImportOptions; import org.hisp.dhis.dxf2.gml.GmlImportService; +import org.hisp.dhis.dxf2.gml.GmlPreProcessingResult; import org.hisp.dhis.scheduling.TaskId; +import org.hisp.dhis.system.notification.NotificationLevel; +import org.hisp.dhis.system.notification.Notifier; +import org.springframework.web.util.HtmlUtils; +import org.xml.sax.SAXParseException; -import javax.xml.transform.TransformerException; -import java.io.IOException; import java.io.InputStream; /** @@ -50,12 +54,15 @@ private String userUid; + // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- private GmlImportService gmlImportService; + private Notifier notifier; + private ImportOptions importOptions; private InputStream inputStream; @@ -64,11 +71,12 @@ // Constructors // ------------------------------------------------------------------------- - public ImportMetaDataGmlTask( String userUid, GmlImportService gmlImportService, + public ImportMetaDataGmlTask( String userUid, GmlImportService gmlImportService, Notifier notifier, ImportOptions importOptions, InputStream inputStream, TaskId taskId ) { this.userUid = userUid; this.gmlImportService = gmlImportService; + this.notifier = notifier; this.importOptions = importOptions; this.inputStream = inputStream; this.taskId = taskId; @@ -83,15 +91,37 @@ { importOptions.setImportStrategy( "update" ); // Force update only for GML import - try - { - gmlImportService.importGml( inputStream, userUid, importOptions, taskId ); - } - catch ( IOException | TransformerException e ) - { - log.error( "Unable to read GML data from input stream", e ); - - throw new RuntimeException( "Failed to parse GML input stream", e ); - } + GmlPreProcessingResult gmlPreProcessingResult = gmlImportService.preProcessGml( inputStream ); + + if ( !gmlPreProcessingResult.isSuccess() ) + { + Throwable throwable = gmlPreProcessingResult.getThrowable(); + String message = createErrorMessage( throwable ); + + notifier.notify( taskId, NotificationLevel.ERROR, message, false ); + log.error( "GML import failed: " + message, throwable ); + + return; + } + + gmlImportService.importGml( gmlPreProcessingResult.getResultMetaData(), userUid, importOptions, taskId ); + } + + private String createErrorMessage( Throwable throwable ) + { + String message = ""; + Throwable rootThrowable = ExceptionUtils.getRootCause( throwable ); + + if ( rootThrowable instanceof SAXParseException ) + { + SAXParseException e = (SAXParseException) rootThrowable; + message += "Syntax error on line " + e.getLineNumber() + ". " + e.getMessage(); + } + else + { + message += rootThrowable.getMessage(); + } + + return HtmlUtils.htmlEscape( message ); } } === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties' --- dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-02-20 11:17:31 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/resources/org/hisp/dhis/importexport/i18n_module.properties 2015-06-05 12:56:34 +0000 @@ -300,6 +300,7 @@ ignored=Ignored conflicts=Conflicts no_conflicts_found=No conflicts found +no_import_summary_available=No summary available type=Type count=Count export_as_xml=Export as XML === modified file 'dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm' --- dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm 2013-03-21 09:15:24 +0000 +++ dhis-2/dhis-web/dhis-web-importexport/src/main/webapp/dhis-web-importexport/importMetaDataSummary.vm 2015-06-05 12:56:34 +0000 @@ -1,4 +1,6 @@

$i18n.getString( "import_summary" )

+ +#if( $summary ) ## ImportSummary can be null on failed GML import (pre-processing fails)

$i18n.getString( "import_count" )

$summary.importCount.imported Imported
@@ -54,3 +56,6 @@ #else

$i18n.getString( "no_conflicts_found" )

#end +#else + $i18n.getString( "no_summary_available" ) +#end \ No newline at end of file