=== modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java 2014-06-12 16:30:25 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetService.java 2014-07-10 16:06:48 +0000 @@ -52,6 +52,8 @@ void writeDataValueSetJson( Set dataSet, Date startDate, Date endDate, Set ous, OutputStream outputStream ); + void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream ); + void writeDataValueSetCsv( Set dataSets, Date startDate, Date endDate, Set orgUnits, Writer writer ); RootNode getDataValueSetTemplate( DataSet dataSet, Period period, List orgUnits, === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetStore.java 2014-07-10 16:06:48 +0000 @@ -38,13 +38,18 @@ import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.Period; +/** + * @author Lars Helge Overland + */ public interface DataValueSetStore { public void writeDataValueSetXml( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, Set dataElements, Set periods, Set orgUnits, OutputStream out ); - public void writeDataValueSetCsv( Set dataElements, - Set periods, Set orgUnits, Writer writer ); + public void writeDataValueSetCsv( Set dataElements, Set periods, + Set orgUnits, Writer writer ); + + void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream ); public void writeDataValueSetJson( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, Set dataElements, Set periods, Set orgUnits, OutputStream out ); === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2014-07-10 12:23:09 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java 2014-07-10 16:06:48 +0000 @@ -217,6 +217,12 @@ { dataValueSetStore.writeDataValueSetJson( null, null, null, null, getDataElements( dataSets ), getPeriods( startDate, endDate ), getOrgUnits( orgUnits ), outputStream ); } + + @Override + public void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream ) + { + dataValueSetStore.writeDataValueSetJson( lastUpdated, outputStream ); + } @Override public void writeDataValueSetCsv( Set dataSets, Date startDate, Date endDate, Set orgUnits, Writer writer ) === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java 2014-07-10 14:16:24 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/SpringDataValueSetStore.java 2014-07-10 16:06:48 +0000 @@ -73,8 +73,10 @@ Set dataElements, Set periods, Set orgUnits, OutputStream out ) { DataValueSet dataValueSet = new StreamingDataValueSet( XMLFactory.getXMLWriter( out ) ); - - writeDataValueSet( dataSet, completeDate, period, orgUnit, dataElements, periods, orgUnits, dataValueSet ); + + SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) ); + + writeDataValueSet( rowSet, dataSet, completeDate, period, orgUnit, dataValueSet ); StreamUtils.closeOutputStream( out ); } @@ -84,29 +86,50 @@ Set dataElements, Set periods, Set orgUnits, OutputStream outputStream ) { DataValueSet dataValueSet = new StreamingJsonDataValueSet( outputStream ); - - writeDataValueSet( dataSet, completeDate, period, orgUnit, dataElements, periods, orgUnits, dataValueSet ); + + SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) ); + + writeDataValueSet( rowSet, dataSet, completeDate, period, orgUnit, dataValueSet ); StreamUtils.closeOutputStream( outputStream ); } @Override + public void writeDataValueSetJson( Date lastUpdated, OutputStream outputStream ) + { + DataValueSet dataValueSet = new StreamingJsonDataValueSet( outputStream ); + + final String sql = + "select de.uid as deuid, pe.startdate, pt.name, ou.uid as ouuid, coc.uid as cocuid, dv.value, dv.storedby, dv.created, dv.lastupdated, dv.comment, dv.followup " + + "from datavalue dv " + + "join dataelement de on (dv.dataelementid=de.dataelementid) " + + "join period pe on (dv.periodid=pe.periodid) " + + "join periodtype pt on (pe.periodtypeid=pt.periodtypeid) " + + "join organisationunit ou on (dv.sourceid=ou.organisationunitid) " + + "join categoryoptioncombo coc on (dv.categoryoptioncomboid=coc.categoryoptioncomboid) " + + "where dv.lastupdated >= '" + DateUtils.getLongDateString( lastUpdated ) + "'"; + + SqlRowSet rowSet = jdbcTemplate.queryForRowSet( sql ); + + writeDataValueSet( rowSet, null, null, null, null, dataValueSet ); + } + + @Override public void writeDataValueSetCsv( Set dataElements, Set periods, Set orgUnits, Writer writer ) { DataValueSet dataValueSet = new StreamingCsvDataValueSet( new CsvWriter( writer, CSV_DELIM ) ); - - writeDataValueSet( null, null, null, null, dataElements, periods, orgUnits, dataValueSet ); + + SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) ); + + writeDataValueSet( rowSet, null, null, null, null, dataValueSet ); } - + //-------------------------------------------------------------------------- // Supportive methods //-------------------------------------------------------------------------- - private void writeDataValueSet( DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, - Set dataElements, Set periods, Set orgUnits, DataValueSet dataValueSet ) + private void writeDataValueSet( SqlRowSet rowSet, DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, DataValueSet dataValueSet ) { - SqlRowSet rowSet = jdbcTemplate.queryForRowSet( getDataValueSql( dataElements, periods, orgUnits ) ); - dataValueSet.setDataSet( dataSet != null ? dataSet.getUid() : null ); dataValueSet.setCompleteDate( getMediumDateString( completeDate ) ); dataValueSet.setPeriod( period != null ? period.getIsoDate() : null ); === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/importsummary/ImportConflict.java 2014-07-10 16:06:48 +0000 @@ -40,6 +40,10 @@ private String value; + public ImportConflict() + { + } + public ImportConflict( String object, String value ) { this.object = object; === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-07 11:22:34 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/DefaultSynchronizationManager.java 2014-07-10 16:06:48 +0000 @@ -30,6 +30,7 @@ import static org.apache.commons.lang.StringUtils.trimToNull; +import java.io.IOException; import java.util.Date; import org.apache.commons.logging.Log; @@ -38,6 +39,9 @@ import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.dxf2.datavalueset.DataValueSetService; +import org.hisp.dhis.dxf2.importsummary.ImportStatus; +import org.hisp.dhis.dxf2.importsummary.ImportSummary; +import org.hisp.dhis.dxf2.utils.ImportSummaryResponseExtractor; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.util.CodecUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -45,12 +49,16 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpRequest; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.support.CronTrigger; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RequestCallback; import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.ResponseExtractor; import org.springframework.web.client.RestTemplate; /** @@ -168,31 +176,62 @@ return false; } - public boolean executeDataSynch() + public ImportSummary executeDataSynch() { AvailabilityStatus availability = isRemoteServerAvailable(); if ( !availability.isAvailable() ) { log.info( "Aborting synch, server not available" ); - return false; + return null; } - Date date = getLastSynchSuccess(); + // --------------------------------------------------------------------- + // Set time for last success to start of process to make data saved + // subsequently part of next synch process without being ignored + // --------------------------------------------------------------------- + + final Date time = getLastSynchSuccess(); - int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( date ); + int lastUpdatedCount = dataValueService.getDataValueCountLastUpdatedAfter( time ); if ( lastUpdatedCount == 0 ) { log.info( "Aborting synch, no new or updated data values" ); - return false; + return null; } - - // Synch - - setLastSynchSuccess(); - - return true; + + final Configuration config = configurationService.getConfiguration(); + + String url = config.getRemoteServerUrl() + "/api/dataValueSets"; + + log.info( "Remote server POST URL: " + url ); + + final RequestCallback requestCallback = new RequestCallback() { + + public void doWithRequest( ClientHttpRequest request ) throws IOException + { + request.getHeaders().setContentType( MediaType.APPLICATION_JSON ); + request.getHeaders().add( HEADER_AUTHORIZATION, CodecUtils.getBasicAuthString( config.getRemoteServerUsername(), config.getRemoteServerPassword() ) ); + dataValueSetService.writeDataValueSetJson( time, request.getBody() ); + } + }; + + ResponseExtractor responseExtractor = new ImportSummaryResponseExtractor(); + + ImportSummary summary = restTemplate.execute( url, HttpMethod.POST, requestCallback, responseExtractor ); + + log.info( summary ); + + if ( summary != null && ImportStatus.SUCCESS.equals( summary.getStatus() ) ) + { + setLastSynchSuccess( time ); + log.info( "Synch successful, setting last success time: " + time ); + } + + return summary; + + //TODO paging of data values } // ------------------------------------------------------------------------- @@ -213,9 +252,9 @@ /** * Sets the time of the last successful synchronization operation. */ - private void setLastSynchSuccess() + private void setLastSynchSuccess( Date time ) { - systemSettingManager.saveSystemSetting( KEY_LAST_SUCCESSFUL_SYNC, new Date() ); + systemSettingManager.saveSystemSetting( KEY_LAST_SUCCESSFUL_SYNC, time ); } /** === modified file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java 2014-07-07 11:00:35 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/synch/SynchronizationManager.java 2014-07-10 16:06:48 +0000 @@ -28,6 +28,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import org.hisp.dhis.dxf2.importsummary.ImportSummary; + /** * @author Lars Helge Overland */ @@ -35,7 +37,7 @@ { AvailabilityStatus isRemoteServerAvailable(); - boolean executeDataSynch(); + ImportSummary executeDataSynch(); void enableDataSynch(); === added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/utils/ImportSummaryResponseExtractor.java 2014-07-10 16:06:48 +0000 @@ -0,0 +1,64 @@ +package org.hisp.dhis.dxf2.utils; + +/* + * Copyright (c) 2004-2014, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.io.IOException; + +import org.hisp.dhis.dxf2.importsummary.ImportSummary; +import org.springframework.http.HttpStatus; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.ResponseExtractor; + +/** + * Converts a response into an ImportSummary instance. + * + * @throws HttpServerErrorException if the response status code is different + * from 200 OK or 201 Created. + * @throws IOException if converting the response into an ImportSummary failed. + * + * @author Lars Helge Overland + */ +public class ImportSummaryResponseExtractor + implements ResponseExtractor +{ + public ImportSummary extractData( ClientHttpResponse response ) throws IOException + { + ImportSummary summary = JacksonUtils.fromJson( response.getBody(), ImportSummary.class ); + + HttpStatus status = response.getStatusCode(); + + if ( !( HttpStatus.CREATED.equals( status ) || HttpStatus.OK.equals( status ) ) ) + { + throw new HttpServerErrorException( status, "Data synch failed on remote server" ); + } + + return summary; + } +} === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java 2014-07-07 10:06:36 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SynchronizationController.java 2014-07-10 16:06:48 +0000 @@ -28,8 +28,16 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_JSON; + +import java.io.IOException; + +import javax.servlet.http.HttpServletResponse; + +import org.hisp.dhis.dxf2.importsummary.ImportSummary; import org.hisp.dhis.dxf2.synch.AvailabilityStatus; import org.hisp.dhis.dxf2.synch.SynchronizationManager; +import org.hisp.dhis.dxf2.utils.JacksonUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; @@ -52,9 +60,19 @@ @Autowired private SynchronizationManager synchronizationManager; + @RequestMapping( method = RequestMethod.POST ) + public void execute( HttpServletResponse response ) + throws IOException + { + ImportSummary summary = synchronizationManager.executeDataSynch(); + + response.setContentType( CONTENT_TYPE_JSON ); + JacksonUtils.toJson( response.getOutputStream(), summary ); + } + @RequestMapping( value = "/availability", method = RequestMethod.GET, produces = "application/json" ) public @ResponseBody AvailabilityStatus isRemoteServerAvailable() { return synchronizationManager.isRemoteServerAvailable(); - } + } }