=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java 2015-10-01 07:51:29 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java 2015-10-07 23:01:57 +0000 @@ -76,6 +76,11 @@ */ private FileResourceDomain domain; + /** + * Current storage status of content. + */ + private FileResourceStorageStatus storageStatus = FileResourceStorageStatus.NONE; + // ------------------------------------------------------------------------- // Constructors // ------------------------------------------------------------------------- @@ -175,6 +180,19 @@ return assigned; } + public void setStorageStatus( FileResourceStorageStatus storageStatus ) + { + this.storageStatus = storageStatus; + } + + @JsonProperty + @JsonView( { DetailedView.class, ExportView.class } ) + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public FileResourceStorageStatus getStorageStatus() + { + return storageStatus; + } + public void setAssigned( boolean assigned ) { this.assigned = assigned; === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 2015-10-07 13:46:41 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 2015-10-07 23:01:57 +0000 @@ -30,6 +30,7 @@ import com.google.common.io.ByteSource; +import java.io.File; import java.net.URI; /** @@ -55,6 +56,16 @@ String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 ); /** + * Save the content of the file to the file store. + * @param key the key to use. Must be unique in the file store. + * @param file the file. The file will be consumed and deleted upon completion. + * @param size the byte length of the file. + * @param contentMd5 the MD5 digest of the file. + * @return the key on success or null if saving failed. + */ + String saveFileResourceContent( String key, File file, long size, String contentMd5 ); + + /** * Delete the content bytes of a file resource. * @param key the key. */ === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 2015-10-06 22:00:49 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 2015-10-07 23:01:57 +0000 @@ -30,6 +30,7 @@ import com.google.common.io.ByteSource; +import java.io.File; import java.net.URI; import java.util.List; @@ -43,7 +44,9 @@ List getFileResources( List uids ); String saveFileResource( FileResource fileResource, ByteSource content ); - + + String saveFileResourceAsync( FileResource fileResource, File file ); + void deleteFileResource( String uid ); ByteSource getFileResourceContent( FileResource fileResource ); === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceStorageStatus.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceStorageStatus.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceStorageStatus.java 2015-10-07 23:01:57 +0000 @@ -0,0 +1,40 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @author Halvdan Hoem Grelland + */ +public enum FileResourceStorageStatus +{ + NONE, // No content stored + PENDING, // In transit to store, not available + FAILED, // Storing the resource failed + STORED // Is available from store +} === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-10-07 13:46:41 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-10-07 23:01:57 +0000 @@ -28,14 +28,16 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import com.google.common.io.ByteSource; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.GenericIdentifiableObjectStore; +import org.hisp.dhis.system.scheduling.Scheduler; import org.springframework.transaction.annotation.Transactional; - -import com.google.common.io.ByteSource; - +import org.springframework.util.concurrent.ListenableFuture; + +import java.io.File; import java.net.URI; import java.util.List; @@ -65,6 +67,20 @@ this.fileResourceContentStore = fileResourceContentStore; } + private Scheduler scheduler; + + public void setScheduler( Scheduler scheduler ) + { + this.scheduler = scheduler; + } + + private FileResourceUploadCallbackProvider uploadCallbackProvider; + + public void setUploadCallbackProvider( FileResourceUploadCallbackProvider uploadCallbackProvider ) + { + this.uploadCallbackProvider = uploadCallbackProvider; + } + // ------------------------------------------------------------------------- // FileResourceService implementation // ------------------------------------------------------------------------- @@ -109,6 +125,26 @@ @Transactional @Override + public String saveFileResourceAsync( FileResource fileResource, File file ) + { + fileResource.setStorageStatus( FileResourceStorageStatus.PENDING ); + fileResourceStore.save( fileResource ); + + String storageKey = getRelativeStorageKey( fileResource ); + + ListenableFuture saveContentTask = + scheduler.executeTask( () -> fileResourceContentStore.saveFileResourceContent( + storageKey, file, fileResource.getContentLength(), fileResource.getContentMd5() ) ); + + String uid = fileResource.getUid(); + + saveContentTask.addCallback( uploadCallbackProvider.getCallback( uid ) ); + + return uid; + } + + @Transactional + @Override public void deleteFileResource( String uid ) { if ( uid == null ) === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/FileResourceUploadCallbackProvider.java 2015-10-07 23:01:57 +0000 @@ -0,0 +1,82 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.concurrent.ListenableFutureCallback; + +/** + * @author Halvdan Hoem Grelland + */ +public class FileResourceUploadCallbackProvider +{ + Log log = LogFactory.getLog( FileResourceUploadCallbackProvider.class ); + + @Autowired + private IdentifiableObjectManager idObjectManager; + + public ListenableFutureCallback getCallback( String fileResourceUid ) + { + return new ListenableFutureCallback() + { + @Override + public void onFailure( Throwable ex ) + { + log.error( "Saving file content failed", ex ); + + FileResource fetchedFileResource = idObjectManager.get( FileResource.class, fileResourceUid ); + fetchedFileResource.setStorageStatus( FileResourceStorageStatus.FAILED ); + idObjectManager.update( fetchedFileResource ); + } + + @Override + public void onSuccess( String result ) + { + log.info( "File content uploaded: " + result ); + + FileResource fetchedFileResource = idObjectManager.get( FileResource.class, fileResourceUid ); + + if ( result != null && fetchedFileResource != null ) + { + fetchedFileResource.setStorageStatus( FileResourceStorageStatus.STORED ); + } + else + { + log.error( "Conflict: content was stored but FileResource with uid: " + fileResourceUid + " could not be found." ); + return; + } + + idObjectManager.update( fetchedFileResource ); + } + }; + } +} === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java 2015-10-07 13:46:41 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java 2015-10-07 23:01:57 +0000 @@ -49,9 +49,11 @@ import org.jclouds.http.HttpRequest; import org.joda.time.Minutes; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -266,6 +268,35 @@ } @Override + public String saveFileResourceContent( String key, File file, long size, String contentMd5 ) + { + Blob blob = blobStore.blobBuilder( key ) + .payload( file ) + .contentLength( size ) + .contentMD5( HashCode.fromString( contentMd5 ) ) + .build(); + + if ( blob == null ) + { + return null; + } + + putBlob( blob ); + + try + { + Files.deleteIfExists( file.toPath() ); + } + catch ( IOException ioe ) + { + // Intentionally ignored. If it can't be deleted, it can't be deleted + log.warn( "Temporary file '" + file.toPath() + "' could not be deleted.", ioe ); + } + + return key; + } + + @Override public void deleteFileResourceContent( String key ) { deleteBlob( key ); @@ -281,7 +312,7 @@ return null; } - HttpRequest httpRequest = null; + HttpRequest httpRequest; try { === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2015-10-07 04:35:59 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2015-10-07 23:01:57 +0000 @@ -578,6 +578,10 @@ + + @@ -588,6 +592,8 @@ + + === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml 2015-09-21 10:09:44 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml 2015-10-07 23:01:57 +0000 @@ -27,8 +27,15 @@ + + + + org.hisp.dhis.fileresource.FileResourceStorageStatus + 12 + + - + org.hisp.dhis.fileresource.FileResourceDomain 12 === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-10-06 22:00:49 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-10-07 23:01:57 +0000 @@ -49,10 +49,13 @@ import org.hisp.dhis.datavalue.DataValue; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.dxf2.utils.InputUtils; +import org.hisp.dhis.dxf2.webmessage.WebMessage; import org.hisp.dhis.dxf2.webmessage.WebMessageException; +import org.hisp.dhis.dxf2.webmessage.responses.FileResourceWebMessageResponse; import org.hisp.dhis.fileresource.FileResource; import org.hisp.dhis.fileresource.FileResourceDomain; import org.hisp.dhis.fileresource.FileResourceService; +import org.hisp.dhis.fileresource.FileResourceStorageStatus; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; @@ -446,14 +449,22 @@ FileResource fileResource = fileResourceService.getFileResource( uid ); - if ( fileResource == null ) + if ( fileResource == null || fileResource.getDomain() != FileResourceDomain.DATA_VALUE ) { - throw new WebMessageException( WebMessageUtils.notFound( "The file resource reference id " + uid + " was not found." ) ); + throw new WebMessageException( WebMessageUtils.notFound( "A data value file resource with id " + uid + " does not exist." ) ); } - if ( fileResource.getDomain() != FileResourceDomain.DATA_VALUE ) + if ( fileResource.getStorageStatus() != FileResourceStorageStatus.STORED ) { - throw new WebMessageException( WebMessageUtils.conflict( "File resource domain must be DATA_VALUE" ) ); + // Special case: + // The FileResource exists and has been tied to this DataValue, however, the underlying file + // content is still in transit (PENDING) to the (most likely external) file store provider. + + WebMessage webMessage = WebMessageUtils.conflict( "The content is being processed and is not available yet. Try again later.", + "The content requested is in transit to the file store and will be available at a later time." ); + webMessage.setResponse( new FileResourceWebMessageResponse( fileResource ) ); + + throw new WebMessageException( webMessage ); } ByteSource content = fileResourceService.getFileResourceContent( fileResource ); === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/FileResourceController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/FileResourceController.java 2015-10-07 13:46:41 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/FileResourceController.java 2015-10-07 23:01:57 +0000 @@ -55,8 +55,11 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.multipart.MultipartFile; +import javax.servlet.ServletContext; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.util.Date; /** @@ -79,6 +82,9 @@ @Autowired private CurrentUserService currentUserService; + @Autowired + private ServletContext servletContext; + // --------------------------------------------------------------------- // Controller methods // --------------------------------------------------------------------- @@ -109,7 +115,7 @@ if ( contentLength <= 0 ) { - throw new WebMessageException( WebMessageUtils.conflict( "Could not read file or file is empty" ) ); + throw new WebMessageException( WebMessageUtils.conflict( "Could not read file or file is empty." ) ); } ByteSource bytes = new ByteSource() @@ -135,14 +141,16 @@ fileResource.setCreated( new Date() ); fileResource.setUser( currentUserService.getCurrentUser() ); - String uid = fileResourceService.saveFileResource( fileResource, bytes ); + File tmpFile = toTempFile( file ); + + String uid = fileResourceService.saveFileResourceAsync( fileResource, tmpFile ); if ( uid == null ) { - throw new WebMessageException( WebMessageUtils.error( "Saving the file failed" ) ); + throw new WebMessageException( WebMessageUtils.error( "Saving the file failed." ) ); } - WebMessage webMessage = new WebMessage( WebMessageStatus.OK, HttpStatus.CREATED ); + WebMessage webMessage = new WebMessage( WebMessageStatus.OK, HttpStatus.ACCEPTED ); webMessage.setResponse( new FileResourceWebMessageResponse( fileResource ) ); return webMessage; @@ -165,4 +173,17 @@ return true; } + + private File toTempFile( MultipartFile multipartFile ) + throws IOException + { + File tempDir = (File) servletContext.getAttribute( ServletContext.TEMPDIR ); + File tmpFile = Files.createTempFile( tempDir.toPath(), "org.hisp.dhis", null ).toFile(); + + System.out.println( "TEMP FILE: " + tmpFile.getAbsolutePath() ); + + multipartFile.transferTo( tmpFile ); + + return tmpFile; + } }