=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/ValueType.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/ValueType.java 2015-09-14 10:39:44 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/ValueType.java 2015-09-14 12:46:20 +0000 @@ -58,7 +58,8 @@ INTEGER_ZERO_OR_POSITIVE( Integer.class ), TRACKER_ASSOCIATE( TrackedEntityInstance.class ), OPTION_SET( String.class ), - USERNAME( String.class ); + USERNAME( String.class ), + FILE_RESOURCE( String.class ); public static final List INTEGER_TYPES = Lists.newArrayList( INTEGER, INTEGER_POSITIVE, INTEGER_NEGATIVE, INTEGER_ZERO_OR_POSITIVE ); @@ -105,4 +106,9 @@ { return this == DATE || this == DATETIME; } + + public boolean isFile() + { + return this == FILE_RESOURCE; + } } === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElement.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElement.java 2015-09-15 09:54:24 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataelement/DataElement.java 2015-09-15 12:13:00 +0000 @@ -232,6 +232,14 @@ } /** + * Indicates whether the value type of this data element is a file (externally stored resource) + */ + public boolean isFileType() + { + return getValueType().isFile(); + } + + /** * Returns the data set of this data element. If this data element has * multiple data sets, the data set with the highest collection frequency is * returned. === added directory 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource' === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResource.java 2015-09-18 16:29:32 +0000 @@ -0,0 +1,201 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonView; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import org.hisp.dhis.common.BaseIdentifiableObject; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.common.view.DetailedView; +import org.hisp.dhis.common.view.ExportView; + +import java.util.UUID; + +/** + * @author Halvdan Hoem Grelland + */ +public class FileResource + extends BaseIdentifiableObject +{ + /** + * MIME type + */ + private String contentType; + + /** + * Byte size of content, non negative + */ + private long contentLength; + + /** + * MD5 digest of content + */ + private String contentMD5; + + /** + * Key used for content storage at external location + */ + private String storageKey; + + /** + * Flag indicating wether the resource is assigned (e.g. to a DataValue) or not. + * Unassigned FileResources are generally safe to delete when reaching a certain age + * (unassigned objects might be in staging). + */ + private boolean assigned = false; + + /** + * The domain which this FileResource belongs to + */ + private FileResourceDomain domain; + + // --------------------------------------------------------------------- + // Constructors + // --------------------------------------------------------------------- + + public FileResource() + { + } + + public FileResource( String name, String contentType, long contentLength, String contentMD5, FileResourceDomain domain ) + { + this.name = name; + this.contentType = contentType; + this.contentLength = contentLength; + this.contentMD5 = contentMD5; + this.domain = domain; + this.storageKey = generateStorageKey(); + } + + // --------------------------------------------------------------------- + // Overrides + // --------------------------------------------------------------------- + + @Override + public boolean haveUniqueNames() + { + return false; + } + + // --------------------------------------------------------------------- + // Getters and setters + // --------------------------------------------------------------------- + + @JsonProperty + @JsonView( { DetailedView.class, ExportView.class } ) + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public String getName() + { + return name; + } + + public void setName( String name ) + { + this.name = name; + } + + @JsonProperty + @JsonView( { DetailedView.class, ExportView.class } ) + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public String getContentType() + { + return contentType; + } + + public void setContentType( String contentType ) + { + this.contentType = contentType; + } + + @JsonProperty + @JsonView( { DetailedView.class, ExportView.class } ) + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public long getContentLength() + { + return contentLength; + } + + public void setContentLength( long contentLength ) + { + this.contentLength = contentLength; + } + + @JsonProperty + @JsonView( { DetailedView.class, ExportView.class } ) + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public String getContentMD5() + { + return contentMD5; + } + + public void setContentMD5( String contentMD5 ) + { + this.contentMD5 = contentMD5; + } + + public String getStorageKey() + { + return storageKey; + } + + public void setStorageKey( String storageKey ) + { + this.storageKey = storageKey; + } + + public boolean isAssigned() + { + return assigned; + } + + public void setAssigned( boolean assigned ) + { + this.assigned = assigned; + } + + public FileResourceDomain getDomain() + { + return domain; + } + + public void setDomain( FileResourceDomain domain ) + { + this.domain = domain; + } + + // --------------------------------------------------------------------- + // Getters and setters + // --------------------------------------------------------------------- + + private String generateStorageKey() + { + return UUID.randomUUID().toString(); + } +} === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java 2015-09-18 16:04:41 +0000 @@ -0,0 +1,41 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.google.common.io.ByteSource; + +/** + * @author Halvdan Hoem Grelland + */ +public interface FileResourceContentStore +{ + ByteSource getFileResourceContent( String key ); + String saveFileResourceContent( String key, ByteSource content, long size, String contentMD5 ); + void deleteFileResourceContent( String key ); +} === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java 2015-09-18 14:41:46 +0000 @@ -0,0 +1,52 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @author Halvdan Hoem Grelland + */ +public enum FileResourceDomain +{ + DATA_VALUE( "dataValue" ); + + /** + * Container name to use when storing blobs of this FileResourceDomain + */ + private String containerName; + + FileResourceDomain( String containerName ) + { + this.containerName = containerName; + } + + public String getContainerName() + { + return containerName; + } +} === added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java 2015-09-10 12:43:38 +0000 @@ -0,0 +1,44 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.google.common.io.ByteSource; + +/** + * @author Halvdan Hoem Grelland + */ +public interface FileResourceService +{ + FileResource getFileResource( String uid ); + String saveFileResource( FileResource fileResource, ByteSource content ); + void deleteFileResource( String uid ); + ByteSource getFileResourceContent( FileResource fileResource ); + boolean fileResourceExists( String uid ); + void updateFileResource( FileResource fileResource ); +} === modified file 'dhis-2/dhis-services/dhis-service-core/pom.xml' --- dhis-2/dhis-services/dhis-service-core/pom.xml 2015-07-20 02:01:37 +0000 +++ dhis-2/dhis-services/dhis-service-core/pom.xml 2015-09-15 13:38:27 +0000 @@ -85,6 +85,18 @@ org.apache.poi poi-ooxml + + org.apache.jclouds + jclouds-all + + + org.apache.jclouds.api + filesystem + + + org.apache.jclouds.provider + aws-s3 + === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datavalue/DefaultDataValueService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datavalue/DefaultDataValueService.java 2015-06-16 05:11:29 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/datavalue/DefaultDataValueService.java 2015-09-18 16:41:31 +0000 @@ -47,6 +47,7 @@ import org.hisp.dhis.dataelement.DataElementCategoryOptionCombo; import org.hisp.dhis.dataelement.DataElementCategoryService; import org.hisp.dhis.dataelement.DataElementOperand; +import org.hisp.dhis.fileresource.FileResourceService; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodType; @@ -95,6 +96,13 @@ this.categoryService = categoryService; } + private FileResourceService fileResourceService; + + public void setFileResourceService( FileResourceService fileResourceService ) + { + this.fileResourceService = fileResourceService; + } + // ------------------------------------------------------------------------- // Basic DataValue // ------------------------------------------------------------------------- @@ -176,6 +184,12 @@ dataValueAuditService.addDataValueAudit( dataValueAudit ); + if ( dataValue.getDataElement().isFileType() ) + { + // TODO Consider for deleteDataValuesBySource and deleteDataValuesByDataElement + fileResourceService.deleteFileResource( dataValue.getValue() ); + } + dataValueStore.deleteDataValue( dataValue ); } === added directory 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource' === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 2015-09-18 16:29:32 +0000 @@ -0,0 +1,182 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.google.common.hash.HashCode; +import com.google.common.io.ByteSource; +import org.apache.commons.io.input.NullInputStream; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.domain.Credentials; +import org.jclouds.domain.Location; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; +import java.util.Properties; + +/** + * @author Halvdan Hoem Grelland + */ +public abstract class BaseJCloudsFileResourceContentStore + implements FileResourceContentStore +{ + Log log = LogFactory.getLog( BaseJCloudsFileResourceContentStore.class ); + + private BlobStore blobStore; + private BlobStoreContext blobStoreContext; + + // ------------------------------------------------------------------------- + // Default config implementations + // ------------------------------------------------------------------------- + + protected Credentials getCredentials() + { + return new Credentials( "Unused", "Unused" ); + } + + protected Properties getOverrides() + { + return new Properties(); + } + + // ------------------------------------------------------------------------- + // Abstract methods + // ------------------------------------------------------------------------- + + protected abstract String getContainer(); + + protected abstract String getJCloudsProviderKey(); + + protected abstract String getLocation(); + + // ------------------------------------------------------------------------- + // Lifecycle management + // ------------------------------------------------------------------------- + + public void init() + { + blobStoreContext = ContextBuilder.newBuilder( getJCloudsProviderKey() ) + .credentials( getCredentials().identity, getCredentials().credential ) + .overrides( getOverrides() ).build( BlobStoreContext.class ); + + blobStore = blobStoreContext.getBlobStore(); + + Optional location = blobStore.listAssignableLocations() + .stream().filter( l -> l.getId().equals( getLocation() ) ).findFirst(); + + blobStore.createContainerInLocation( location.isPresent() ? location.get() : null, getContainer() ); + } + + public void cleanUp() + { + blobStoreContext.close(); + } + + // ------------------------------------------------------------------------- + // FileResourceContentStore implementation + // ------------------------------------------------------------------------- + + public ByteSource getFileResourceContent( String key ) + { + final Blob blob = getBlob( key ); + + if ( blob == null ) + { + return null; + } + + return new ByteSource() + { + @Override + public InputStream openStream() + { + try + { + return blob.getPayload().openStream(); + } + catch ( IOException e ) + { + return new NullInputStream( 0 ); + } + } + }; + } + + public String saveFileResourceContent( String key, ByteSource content, long size, String contentMD5 ) + { + Blob blob = createBlob( key, content, size, contentMD5 ); + + if ( blob == null ) + { + return null; + } + + putBlob( blob ); + + return key; + } + + public void deleteFileResourceContent( String key ) + { + deleteBlob( key ); + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + private Blob getBlob( String key ) + { + return blobStore.getBlob( getContainer(), key ); + } + + private void deleteBlob( String key ) + { + blobStore.removeBlob( getContainer(), key ); + } + + private String putBlob( Blob blob ) + { + return blobStore.putBlob( getContainer(), blob ); + } + + private Blob createBlob( String key, ByteSource content, long size, String contentMD5 ) + { + return blobStore.blobBuilder( key ) + .payload( content ) + .contentLength( size ) + .contentMD5( HashCode.fromString( contentMD5 ) ) + .build(); + } +} === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 2015-09-18 16:41:31 +0000 @@ -0,0 +1,217 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.external.location.LocationManager; +import org.hisp.dhis.hibernate.HibernateConfigurationProvider; +import org.jclouds.domain.Credentials; +import org.jclouds.filesystem.reference.FilesystemConstants; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * TODO Merge with BaseJCloudsFileResourceContentStore ? + * @author Halvdan Hoem Grelland + */ +public class DefaultFileResourceContentStore + extends BaseJCloudsFileResourceContentStore +{ + private static final Log log = LogFactory.getLog( DefaultFileResourceContentStore.class ); + + // ------------------------------------------------------------------------- + // Provider constants + // ------------------------------------------------------------------------- + + private static final String JCLOUDS_PROVIDER_KEY_FILESYSTEM = "filesystem"; + private static final String JCLOUDS_PROVIDER_KEY_AWS_S3 = "aws-s3"; + + private static final List AVAILABLE_PROVIDERS = new ArrayList() {{ + addAll( Arrays.asList( JCLOUDS_PROVIDER_KEY_FILESYSTEM, JCLOUDS_PROVIDER_KEY_AWS_S3 ) ); + }}; + + // ------------------------------------------------------------------------- + // Property keys + // ------------------------------------------------------------------------- + + private static final String FILESTORE_CONFIG_NAMESPACE = "filestore"; + + private static final String KEY_FILESTORE_PROVIDER = FILESTORE_CONFIG_NAMESPACE + ".provider"; + private static final String KEY_FILESTORE_CONTAINER = FILESTORE_CONFIG_NAMESPACE + ".container"; + private static final String KEY_FILESTORE_LOCATION = FILESTORE_CONFIG_NAMESPACE + ".location"; + private static final String KEY_FILESTORE_IDENTITY = FILESTORE_CONFIG_NAMESPACE + ".identity"; + private static final String KEY_FILESTORE_SECRET = FILESTORE_CONFIG_NAMESPACE + ".secret"; + + // ------------------------------------------------------------------------- + // Defaults + // ------------------------------------------------------------------------- + + private static final String DEFAULT_PROVIDER = "filesystem"; + private static final String DEFAULT_CONTAINER = "dhis2_filestore"; + + // ------------------------------------------------------------------------- + // Configuration + // ------------------------------------------------------------------------- + + private Map filestoreConfiguration; + + private String provider; + private String container; + private Credentials credentials; + private String location; + private Properties overrides = new Properties(); + + // ------------------------------------------------------------------------- + // Dependencies + // ------------------------------------------------------------------------- + + private LocationManager locationManager; + + public void setLocationManager( LocationManager locationManager ) + { + this.locationManager = locationManager; + } + + private HibernateConfigurationProvider configurationProvider; + + public void setConfigurationProvider( HibernateConfigurationProvider configurationProvider ) + { + this.configurationProvider = configurationProvider; + } + + // ------------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------------- + + public void init() + { + filestoreConfiguration = configurationProvider.getConfiguration().getProperties() + .entrySet().stream().filter( + p -> ( (String) p.getKey() ).startsWith( FILESTORE_CONFIG_NAMESPACE ) ) + .collect( Collectors.toMap( + p -> StringUtils.strip( (String) p.getKey() ), + p -> StringUtils.strip( (String) p.getValue() ) + ) ); + + provider = filestoreConfiguration.getOrDefault( KEY_FILESTORE_PROVIDER, DEFAULT_PROVIDER ); + + if ( !AVAILABLE_PROVIDERS.contains( provider ) ) + { + log.info( "Ignored unsupported file store provider '" + provider + "', falling back to file system." ); + provider = DEFAULT_PROVIDER; + } + + container = filestoreConfiguration.getOrDefault( KEY_FILESTORE_CONTAINER, DEFAULT_CONTAINER ); + + location = filestoreConfiguration.getOrDefault( KEY_FILESTORE_LOCATION, null ); + + switch ( provider ) + { + case JCLOUDS_PROVIDER_KEY_FILESYSTEM: + configureFilesystemProvider(); + break; + case JCLOUDS_PROVIDER_KEY_AWS_S3: + configureAWSS3Provider(); + break; + default: + throw new IllegalArgumentException( "The filestore provider " + provider + " is not supported." ); + } + + super.init(); + } + + public void cleanUp() + { + super.cleanUp(); + } + + // ------------------------------------------------------------------------- + // Configuration implementation + // ------------------------------------------------------------------------- + + @Override + protected Properties getOverrides() + { + return overrides; + } + + @Override + protected Credentials getCredentials() + { + return credentials; + } + + @Override + protected String getContainer() + { + return container; + } + + @Override + protected String getLocation() + { + return location; + } + + @Override + protected String getJCloudsProviderKey() + { + return provider; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + private void configureFilesystemProvider() + { + overrides.setProperty( FilesystemConstants.PROPERTY_BASEDIR, locationManager.getExternalDirectoryPath() ); + credentials = super.getCredentials(); + log.info( "File system filestore provider configured." ); + } + + private void configureAWSS3Provider() + { + credentials = new Credentials( filestoreConfiguration.getOrDefault( + KEY_FILESTORE_IDENTITY, "" ), filestoreConfiguration.getOrDefault( KEY_FILESTORE_SECRET, "" ) ); + log.info( "AWS S3 filestore provider configured." ); + + if ( credentials.identity.isEmpty() || credentials.credential.isEmpty() ) + { + log.info( "AWS S3 configured with empty credentials. Authentication will fail" ); + } + } +} === added file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-09-18 16:06:33 +0000 @@ -0,0 +1,142 @@ +package org.hisp.dhis.fileresource; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.google.common.io.ByteSource; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.common.GenericIdentifiableObjectStore; +import org.springframework.transaction.annotation.Transactional; + +/** + * @author Halvdan Hoem Grelland + */ +public class DefaultFileResourceService + implements FileResourceService +{ + + private static final Log log = LogFactory.getLog( DefaultFileResourceService.class ); + + // ------------------------------------------------------------------------- + // Dependencies + // ------------------------------------------------------------------------- + + private GenericIdentifiableObjectStore fileResourceStore; + + public void setFileResourceStore( GenericIdentifiableObjectStore fileResourceStore ) + { + this.fileResourceStore = fileResourceStore; + } + + private FileResourceContentStore fileResourceContentStore; + + public void setFileResourceContentStore( FileResourceContentStore fileResourceContentStore ) + { + this.fileResourceContentStore = fileResourceContentStore; + } + + // ------------------------------------------------------------------------- + // FileResourceService implementation + // ------------------------------------------------------------------------- + + @Override + public FileResource getFileResource( String uid ) + { + return fileResourceStore.getByUid( uid ); + } + + @Transactional + @Override + public String saveFileResource( FileResource fileResource, ByteSource content ) + { + String storageKey = getRelativeStorageKey( fileResource ); + + String key = fileResourceContentStore.saveFileResourceContent( + storageKey, content, fileResource.getContentLength(), fileResource.getContentMD5() ); + + if ( key == null ) + { + log.debug( "Failed saving content for FileResource" ); + return null; + } + + int id = fileResourceStore.save( fileResource ); + + if ( id <= 0 ) + { + log.debug( "Failed persisting the FileResource: " + fileResource.getName() ); + return null; + } + + return fileResource.getUid(); + } + + @Transactional + @Override + public void deleteFileResource( String uid ) + { + FileResource fileResource = fileResourceStore.getByUid( uid ); + + if ( fileResource == null ) + { + return; + } + + fileResourceContentStore.deleteFileResourceContent( getRelativeStorageKey( fileResource ) ); + fileResourceStore.delete( fileResource ); + } + + @Override + public ByteSource getFileResourceContent( FileResource fileResource ) + { + return fileResourceContentStore.getFileResourceContent( getRelativeStorageKey( fileResource ) ); + } + + @Override + public boolean fileResourceExists( String uid ) + { + return fileResourceStore.getByUid( uid ) != null; + } + + @Override + public void updateFileResource( FileResource fileResource ) + { + fileResourceStore.update( fileResource ); + } + + // --------------------------------------------------------------------- + // Supportive methods + // --------------------------------------------------------------------- + + private String getRelativeStorageKey( FileResource fileResource ) + { + return StringUtils.prependIfMissing( fileResource.getStorageKey(), fileResource.getDomain().getContainerName() + "/" ); + } +} === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2015-09-14 11:56:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/META-INF/dhis/beans.xml 2015-09-18 16:29:32 +0000 @@ -563,8 +563,24 @@ + + + + + + + + + + + + + + + @@ -576,6 +592,7 @@ + === added directory 'dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource' === added directory 'dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate' === added file 'dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml' --- dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/resources/org/hisp/dhis/fileresource/hibernate/FileResource.hbm.xml 2015-09-16 15:10:50 +0000 @@ -0,0 +1,41 @@ + +] + > + + + + + + + + + + &identifiableProperties; + + + + + + + + + + + + + + + + + org.hisp.dhis.fileresource.FileResourceDomain + 12 + + + + + + + === added file 'dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/responses/FileResourceWebMessageResponse.java' --- dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/responses/FileResourceWebMessageResponse.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/responses/FileResourceWebMessageResponse.java 2015-09-02 15:39:01 +0000 @@ -0,0 +1,57 @@ +package org.hisp.dhis.dxf2.webmessage.responses; + +/* + * Copyright (c) 2004-2015, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.dxf2.webmessage.AbstractWebMessageResponse; +import org.hisp.dhis.fileresource.FileResource; + +/** + * @author Halvdan Hoem Grelland + */ +public class FileResourceWebMessageResponse + extends AbstractWebMessageResponse +{ + private FileResource fileResource; + + public FileResourceWebMessageResponse( FileResource fileResource ) + { + this.setResponseType( FileResource.class.getSimpleName() ); + this.fileResource = fileResource; + } + + @JsonProperty + @JacksonXmlProperty( namespace = DxfNamespaces.DXF_2_0 ) + public FileResource getFileResource() + { + return fileResource; + } +} === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ValidationUtils.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ValidationUtils.java 2015-09-15 05:39:53 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ValidationUtils.java 2015-09-15 12:13:00 +0000 @@ -32,6 +32,7 @@ import org.apache.commons.validator.routines.DateValidator; import org.apache.commons.validator.routines.EmailValidator; import org.apache.commons.validator.routines.UrlValidator; +import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.commons.util.TextUtils; @@ -375,6 +376,11 @@ return "value_not_valid_datetime"; } + if ( ValueType.FILE_RESOURCE == valueType && !CodeGenerator.isValidCode( value ) ) + { + return "value_not_valid_file_resource_uid"; + } + return null; } === modified file 'dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java' --- dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-09-04 10:49:08 +0000 +++ dhis-2/dhis-web/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/DataValueController.java 2015-09-18 15:47:54 +0000 @@ -28,18 +28,13 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO; -import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS; -import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_ORGANISATION_UNITS; -import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_PERIODS; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import javax.servlet.http.HttpServletResponse; - +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.NullInputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.http.entity.ContentType; import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataelement.DataElement; @@ -48,7 +43,13 @@ import org.hisp.dhis.dataset.DataSetService; import org.hisp.dhis.datavalue.DataValue; import org.hisp.dhis.datavalue.DataValueService; +import org.hisp.dhis.dxf2.webmessage.WebMessage; import org.hisp.dhis.dxf2.webmessage.WebMessageException; +import org.hisp.dhis.dxf2.webmessage.WebMessageStatus; +import org.hisp.dhis.dxf2.webmessage.responses.FileResourceWebMessageResponse; +import org.hisp.dhis.fileresource.FileResource; +import org.hisp.dhis.fileresource.FileResourceDomain; +import org.hisp.dhis.fileresource.FileResourceService; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; @@ -59,12 +60,30 @@ import org.hisp.dhis.webapi.utils.InputUtils; import org.hisp.dhis.webapi.utils.WebMessageUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; +import org.springframework.util.InvalidMimeTypeException; +import org.springframework.util.MimeType; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO; +import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS; +import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_ORGANISATION_UNITS; +import static org.hisp.dhis.setting.SystemSettingManager.KEY_DATA_IMPORT_STRICT_PERIODS; /** * @author Lars Helge Overland @@ -103,6 +122,9 @@ @Autowired private InputUtils inputUtils; + @Autowired + private FileResourceService fileResourceService; + // --------------------------------------------------------------------- // POST // --------------------------------------------------------------------- @@ -118,7 +140,8 @@ @RequestParam String ou, @RequestParam( required = false ) String value, @RequestParam( required = false ) String comment, - @RequestParam( required = false ) boolean followUp, HttpServletResponse response ) throws WebMessageException + @RequestParam( required = false ) boolean followUp, HttpServletResponse response ) + throws WebMessageException { boolean strictPeriods = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_STRICT_PERIODS, false ); boolean strictCategoryOptionCombos = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS, false ); @@ -129,65 +152,17 @@ // Input validation // --------------------------------------------------------------------- - DataElement dataElement = idObjectManager.get( DataElement.class, de ); - - if ( dataElement == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal data element identifier: " + de ) ); - } - - DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo( co ); - - if ( categoryOptionCombo == null ) - { - if ( requireCategoryOptionCombo ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Category option combo is required but is not specified" ) ); - } - else if ( co != null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal category option combo identifier: " + co ) ); - } - else - { - categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); - } - } - - DataElementCategoryOptionCombo attributeOptionCombo = inputUtils.getAttributeOptionCombo( cc, cp ); - - if ( attributeOptionCombo == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal attribute option combo identifier: " + cc + " " + cp ) ); - } - - Period period = PeriodType.getPeriodFromIsoString( pe ); - - if ( period == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal period identifier: " + pe ) ); - } - - OrganisationUnit organisationUnit = idObjectManager.get( OrganisationUnit.class, ou ); - - if ( organisationUnit == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal organisation unit identifier: " + ou ) ); - } - - boolean inUserHierarchy = organisationUnitService.isInUserHierarchy( organisationUnit ); - - if ( !inUserHierarchy ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Organisation unit is not in the hierarchy of the current user: " + ou ) ); - } - - boolean invalidFuturePeriod = period.isFuture() && dataElement.getOpenFuturePeriods() <= 0; - - if ( invalidFuturePeriod ) - { - throw new WebMessageException( WebMessageUtils.conflict( "One or more data sets for data element does not allow future periods: " + de ) ); - } + DataElement dataElement = getAndValidateDataElement( de ); + + DataElementCategoryOptionCombo categoryOptionCombo = getAndValidateCategoryOptionCombo( co, requireCategoryOptionCombo ); + + DataElementCategoryOptionCombo attributeOptionCombo = getAndValidateAttributeOptionCombo( cc, cp ); + + Period period = getAndValidatePeriod( pe ); + + OrganisationUnit organisationUnit = getAndValidateOrganisationUnit( ou ); + + validateInvalidFuturePeriod( period, dataElement ); String valueValid = ValidationUtils.dataValueIsValid( value, dataElement ); @@ -229,9 +204,29 @@ // Locking validation // --------------------------------------------------------------------- - if ( dataSetService.isLocked( dataElement, period, organisationUnit, null ) ) + validateDataSetNotLocked( dataElement, period, organisationUnit ); + + // --------------------------------------------------------------------- + // Deal with file resource + // --------------------------------------------------------------------- + + FileResource fileResource = null; + + if ( dataElement.getValueType() == ValueType.FILE_RESOURCE ) { - throw new WebMessageException( WebMessageUtils.conflict( "Data set is locked" ) ); + fileResource = fileResourceService.getFileResource( value ); + + if ( fileResource == null || fileResource.getDomain() != FileResourceDomain.DATA_VALUE ) + { + throw new WebMessageException( WebMessageUtils.notFound( FileResource.class, value ) ); + } + + if ( fileResource.isAssigned() ) + { + throw new WebMessageException( WebMessageUtils.conflict( "File resource is already assigned or is linked to another data value" ) ); + } + + fileResource.setAssigned( true ); } // --------------------------------------------------------------------- @@ -268,6 +263,11 @@ if ( value != null ) { + if ( dataElement.isFileType() ) + { + fileResourceService.deleteFileResource( dataValue.getValue() ); + } + dataValue.setValue( StringUtils.trimToNull( value ) ); } @@ -286,6 +286,11 @@ dataValueService.updateDataValue( dataValue ); } + + if ( fileResource != null ) + { + fileResourceService.updateFileResource( fileResource ); + } } // --------------------------------------------------------------------- @@ -300,71 +305,28 @@ @RequestParam( required = false ) String cc, @RequestParam( required = false ) String cp, @RequestParam String pe, - @RequestParam String ou, HttpServletResponse response ) throws WebMessageException + @RequestParam String ou, HttpServletResponse response ) + throws WebMessageException { // --------------------------------------------------------------------- // Input validation // --------------------------------------------------------------------- - DataElement dataElement = idObjectManager.get( DataElement.class, de ); - - if ( dataElement == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal data element identifier: " + de ) ); - } - - DataElementCategoryOptionCombo categoryOptionCombo; - - if ( co != null ) - { - categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo( co ); - } - else - { - categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); - } - - if ( categoryOptionCombo == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal category option combo identifier: " + co ) ); - } - - DataElementCategoryOptionCombo attributeOptionCombo = inputUtils.getAttributeOptionCombo( cc, cp ); - - if ( attributeOptionCombo == null ) - { - return; - } - - Period period = PeriodType.getPeriodFromIsoString( pe ); - - if ( period == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal period identifier: " + pe ) ); - } - - OrganisationUnit organisationUnit = idObjectManager.get( OrganisationUnit.class, ou ); - - if ( organisationUnit == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal organisation unit identifier: " + ou ) ); - } - - boolean isInHierarchy = organisationUnitService.isInUserHierarchy( organisationUnit ); - - if ( !isInHierarchy ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Organisation unit is not in the hierarchy of the current user: " + ou ) ); - } + DataElement dataElement = getAndValidateDataElement( de ); + + DataElementCategoryOptionCombo categoryOptionCombo = getAndValidateCategoryOptionCombo( co, false ); + + DataElementCategoryOptionCombo attributeOptionCombo = getAndValidateAttributeOptionCombo( cc, cp ); + + Period period = getAndValidatePeriod( pe ); + + OrganisationUnit organisationUnit = getAndValidateOrganisationUnit( ou ); // --------------------------------------------------------------------- // Locking validation // --------------------------------------------------------------------- - if ( dataSetService.isLocked( dataElement, period, organisationUnit, null ) ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Data set is locked" ) ); - } + validateDataSetNotLocked( dataElement, period, organisationUnit ); // --------------------------------------------------------------------- // Delete data value @@ -392,71 +354,28 @@ @RequestParam( required = false ) String cp, @RequestParam String pe, @RequestParam String ou, - Model model, HttpServletResponse response ) throws WebMessageException + Model model, HttpServletResponse response ) + throws WebMessageException { // --------------------------------------------------------------------- // Input validation // --------------------------------------------------------------------- - DataElement dataElement = idObjectManager.get( DataElement.class, de ); - - if ( dataElement == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal data element identifier: " + de ) ); - } - - DataElementCategoryOptionCombo categoryOptionCombo; - - if ( co != null ) - { - categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo( co ); - } - else - { - categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); - } - - if ( categoryOptionCombo == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal category option combo identifier: " + co ) ); - } - - DataElementCategoryOptionCombo attributeOptionCombo = inputUtils.getAttributeOptionCombo( cc, cp ); - - if ( attributeOptionCombo == null ) - { - return null; - } - - Period period = PeriodType.getPeriodFromIsoString( pe ); - - if ( period == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal period identifier: " + pe ) ); - } - - OrganisationUnit organisationUnit = idObjectManager.get( OrganisationUnit.class, ou ); - - if ( organisationUnit == null ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Illegal organisation unit identifier: " + ou ) ); - } - - boolean isInHierarchy = organisationUnitService.isInUserHierarchy( organisationUnit ); - - if ( !isInHierarchy ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Organisation unit is not in the hierarchy of the current user: " + ou ) ); - } + DataElement dataElement = getAndValidateDataElement( de ); + + DataElementCategoryOptionCombo categoryOptionCombo = getAndValidateCategoryOptionCombo( co, false ); + + DataElementCategoryOptionCombo attributeOptionCombo = getAndValidateAttributeOptionCombo( cc, cp ); + + Period period = getAndValidatePeriod( pe ); + + OrganisationUnit organisationUnit = getAndValidateOrganisationUnit( ou ); // --------------------------------------------------------------------- // Locking validation // --------------------------------------------------------------------- - if ( dataSetService.isLocked( dataElement, period, organisationUnit, null ) ) - { - throw new WebMessageException( WebMessageUtils.conflict( "Data set is locked" ) ); - } + validateDataSetNotLocked( dataElement, period, organisationUnit ); // --------------------------------------------------------------------- // Get data value @@ -476,4 +395,357 @@ return "value"; } + + // --------------------------------------------------------------------- + // POST file + // --------------------------------------------------------------------- + + @PreAuthorize( "hasRole('ALL') or hasRole('F_DATAVALUE_ADD')" ) + @RequestMapping( value = "/files", method = RequestMethod.POST ) + public @ResponseBody WebMessage saveDataValueFileResource( + @RequestParam String de, + @RequestParam( required = false ) String co, + @RequestParam( required = false ) String cc, + @RequestParam( required = false ) String cp, + @RequestParam String pe, + @RequestParam String ou, + @RequestParam( value = "file", required = true ) MultipartFile multipartFile, + HttpServletResponse response ) + throws WebMessageException, IOException + { + + boolean strictPeriods = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_STRICT_PERIODS, false ); + boolean strictCategoryOptionCombos = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS, false ); + boolean strictOrgUnits = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_STRICT_ORGANISATION_UNITS, false ); + boolean requireCategoryOptionCombo = (Boolean) systemSettingManager.getSystemSetting( KEY_DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO, false ); + + // --------------------------------------------------------------------- + // Input validation + // --------------------------------------------------------------------- + + DataElement dataElement = getAndValidateDataElement( de ); + + DataElementCategoryOptionCombo categoryOptionCombo = getAndValidateCategoryOptionCombo( co, requireCategoryOptionCombo ); + + DataElementCategoryOptionCombo attributeOptionCombo = getAndValidateAttributeOptionCombo( cc, cp ); + + Period period = getAndValidatePeriod( pe ); + + OrganisationUnit organisationUnit = getAndValidateOrganisationUnit( ou ); + + validateInvalidFuturePeriod( period, dataElement ); + + if ( multipartFile == null || multipartFile.isEmpty() ) + { + throw new WebMessageException( WebMessageUtils.conflict( "File is missing", + "The multipart request didn't contain a file or the file was empty." ) ); + } + + // --------------------------------------------------------------------- + // Optional constraints + // --------------------------------------------------------------------- + + if ( strictPeriods && !dataElement.getPeriodTypes().contains( period.getPeriodType() ) ) + { + throw new WebMessageException( WebMessageUtils.conflict( + "Period type of period: " + period.getIsoDate() + " not valid for data element: " + dataElement.getUid() ) ); + } + + if ( strictCategoryOptionCombos && !dataElement.getCategoryCombo().getOptionCombos().contains( categoryOptionCombo ) ) + { + throw new WebMessageException( WebMessageUtils.conflict( + "Category option combo: " + categoryOptionCombo.getUid() + " must be part of category combo of data element: " + dataElement.getUid() ) ); + } + + if ( strictOrgUnits && !dataElement.hasDataSetOrganisationUnit( organisationUnit ) ) + { + throw new WebMessageException( WebMessageUtils.conflict( + "Data element: " + dataElement.getUid() + " must be assigned through data sets to organisation unit: " + organisationUnit.getUid() ) ); + } + + // --------------------------------------------------------------------- + // Locking validation + // --------------------------------------------------------------------- + + validateDataSetNotLocked( dataElement, period, organisationUnit ); + + // --------------------------------------------------------------------- + // Validate and assemble FileResource + // --------------------------------------------------------------------- + + String filename = StringUtils.defaultIfBlank( FilenameUtils.getName( multipartFile.getOriginalFilename() ), "untitled" ); + + String contentType = multipartFile.getContentType(); + contentType = isValidContentType( contentType ) ? contentType : ContentType.APPLICATION_OCTET_STREAM.toString(); + + long contentLength = multipartFile.getSize(); + + if ( contentLength <= 0 ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Could not read file or file is empty" ) ); + } + + ByteSource content = new ByteSource() + { + @Override + public InputStream openStream() + { + try + { + return multipartFile.getInputStream(); + } + catch ( IOException e ) + { + return new NullInputStream( 0 ); + } + } + }; + + String contentMD5 = content.hash( Hashing.md5() ).toString(); // TODO Consider letting filestore create the hash + + FileResource fileResource = new FileResource( filename, contentType, contentLength, contentMD5, FileResourceDomain.DATA_VALUE ); + fileResource.setAssigned( false ); + fileResource.setCreated( new Date() ); + fileResource.setUser( currentUserService.getCurrentUser() ); + + // --------------------------------------------------------------------- + // Save file resource + // --------------------------------------------------------------------- + + String uid = fileResourceService.saveFileResource( fileResource, content ); + + if ( uid == null ) + { + throw new WebMessageException( WebMessageUtils.error( "Saving the file failed" ) ); + } + + WebMessage webMessage = new WebMessage( WebMessageStatus.OK, HttpStatus.CREATED ); + webMessage.setResponse( new FileResourceWebMessageResponse( fileResource ) ); + + return webMessage; + } + + // --------------------------------------------------------------------- + // GET file + // --------------------------------------------------------------------- + + @RequestMapping( value = "/files", method = RequestMethod.GET ) + public void getDataValueFile( + @RequestParam String de, + @RequestParam( required = false ) String co, + @RequestParam( required = false ) String cc, + @RequestParam( required = false ) String cp, + @RequestParam String pe, + @RequestParam String ou, HttpServletResponse response ) + throws WebMessageException + { + // --------------------------------------------------------------------- + // Input validation + // --------------------------------------------------------------------- + + DataElement dataElement = getAndValidateDataElement( de ); + + if ( !dataElement.isFileType() ) + { + throw new WebMessageException( WebMessageUtils.conflict( "DataElement must be of type file" ) ); + } + + DataElementCategoryOptionCombo categoryOptionCombo = getAndValidateCategoryOptionCombo( co, false ); + + DataElementCategoryOptionCombo attributeOptionCombo = getAndValidateAttributeOptionCombo( cc, cp ); + + Period period = getAndValidatePeriod( pe ); + + OrganisationUnit organisationUnit = getAndValidateOrganisationUnit( ou ); + + // --------------------------------------------------------------------- + // Locking validation + // --------------------------------------------------------------------- + + validateDataSetNotLocked( dataElement, period, organisationUnit ); + + // --------------------------------------------------------------------- + // Get data value + // --------------------------------------------------------------------- + + DataValue dataValue = dataValueService.getDataValue( dataElement, period, organisationUnit, categoryOptionCombo, attributeOptionCombo ); + + if ( dataValue == null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Data value does not exist" ) ); + } + + // --------------------------------------------------------------------- + // Get file resource + // --------------------------------------------------------------------- + + String uid = dataValue.getValue(); + + FileResource fileResource = fileResourceService.getFileResource( uid ); + + if ( fileResource == null ) + { + throw new WebMessageException( WebMessageUtils.notFound( "The file resource reference id " + uid + " was not found." ) ); + } + + if ( fileResource.getDomain() != FileResourceDomain.DATA_VALUE ) + { + throw new WebMessageException( WebMessageUtils.conflict( "File resource domain must be of type DATA_VALUE" ) ); + } + + ByteSource content = fileResourceService.getFileResourceContent( fileResource ); + + if ( content == null ) + { + throw new WebMessageException( WebMessageUtils.notFound( "The referenced file could not be found" ) ); + } + + // --------------------------------------------------------------------- + // Build response and return + // --------------------------------------------------------------------- + + response.setContentType( fileResource.getContentType() ); + response.setContentLength( Math.round( fileResource.getContentLength() ) ); + response.setHeader( HttpHeaders.CONTENT_DISPOSITION, "filename=" + fileResource.getName() ); + + InputStream inputStream = null; + + try + { + inputStream = content.openStream(); + IOUtils.copyLarge( inputStream, response.getOutputStream() ); + } + catch ( IOException e ) + { + throw new WebMessageException( WebMessageUtils.error( "Failed fetching the file from storage", + "There was an exception when trying to fetch the file from the storage backend. " + + "Depending on the provider the root cause could be network or file system related." ) ); + } + finally + { + IOUtils.closeQuietly( inputStream ); + } + } + + // --------------------------------------------------------------------- + // Supportive methods + // --------------------------------------------------------------------- + + private DataElement getAndValidateDataElement( String de ) + throws WebMessageException + { + DataElement dataElement = idObjectManager.get( DataElement.class, de ); + + if ( dataElement == null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Illegal data element identifier: " + de ) ); + } + + return dataElement; + } + + private DataElementCategoryOptionCombo getAndValidateCategoryOptionCombo( String co, boolean requireCategoryOptionCombo ) + throws WebMessageException + { + DataElementCategoryOptionCombo categoryOptionCombo = categoryService.getDataElementCategoryOptionCombo( co ); + + if ( categoryOptionCombo == null ) + { + if ( requireCategoryOptionCombo ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Category option combo is required but is not specified" ) ); + } + else if ( co != null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Illegal category option combo identifier: " + co ) ); + } + else + { + categoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo(); + } + } + + return categoryOptionCombo; + } + + private DataElementCategoryOptionCombo getAndValidateAttributeOptionCombo( String cc, String cp ) + throws WebMessageException + { + DataElementCategoryOptionCombo attributeOptionCombo = inputUtils.getAttributeOptionCombo( cc, cp ); + + if ( attributeOptionCombo == null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Illegal attribute option combo identifier: " + cc + " " + cp ) ); + } + + return attributeOptionCombo; + } + + private Period getAndValidatePeriod( String pe ) + throws WebMessageException + { + Period period = PeriodType.getPeriodFromIsoString( pe ); + + if ( period == null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Illegal period identifier: " + pe ) ); + } + + return period; + } + + private OrganisationUnit getAndValidateOrganisationUnit( String ou ) + throws WebMessageException + { + OrganisationUnit organisationUnit = idObjectManager.get( OrganisationUnit.class, ou ); + + if ( organisationUnit == null ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Illegal organisation unit identifier: " + ou ) ); + } + + boolean isInHierarchy = organisationUnitService.isInUserHierarchy( organisationUnit ); + + if ( !isInHierarchy ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Organisation unit is not in the hierarchy of the current user: " + ou ) ); + } + + return organisationUnit; + } + + private void validateInvalidFuturePeriod( Period period, DataElement dataElement ) + throws WebMessageException + { + boolean invalidFuturePeriod = period.isFuture() && dataElement.getOpenFuturePeriods() <= 0; + + if ( invalidFuturePeriod ) + { + throw new WebMessageException( + WebMessageUtils.conflict( "One or more data sets for data element does not allow future periods: " + dataElement.getUid() ) ); + } + } + + private void validateDataSetNotLocked( DataElement dataElement, Period period, OrganisationUnit organisationUnit ) + throws WebMessageException + { + if ( dataSetService.isLocked( dataElement, period, organisationUnit, null ) ) + { + throw new WebMessageException( WebMessageUtils.conflict( "Data set is locked" ) ); + } + } + + private boolean isValidContentType( String contentType ) + { + try + { + MimeType.valueOf( contentType ); + } + catch ( InvalidMimeTypeException e ) + { + return false; + } + + return true; + } } === modified file 'dhis-2/pom.xml' --- dhis-2/pom.xml 2015-09-02 08:35:29 +0000 +++ dhis-2/pom.xml 2015-09-15 14:50:16 +0000 @@ -489,6 +489,12 @@ org.springframework.security spring-security-openid ${spring.security.version} + + + com.google.inject + guice + + @@ -791,6 +797,23 @@ 9.3-1102-jdbc41 + + + org.apache.jclouds + jclouds-all + ${jclouds.version} + + + org.apache.jclouds.api + filesystem + ${jclouds.version} + + + org.apache.jclouds.provider + aws-s3 + ${jclouds.version} + + net.sf.jasperreports @@ -997,6 +1020,12 @@ org.openid4java openid4java 0.9.8 + + + com.google.inject + guice + + xml-apis @@ -1021,6 +1050,7 @@ 2.3.16.3 4.2.20.Final 4.3.2.Final + 1.9.1 3.18.1-GA 2.5.3 1.6.6