=== removed file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 2015-09-18 18:01:58 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/BaseJCloudsFileResourceContentStore.java 1970-01-01 00:00:00 +0000 @@ -1,183 +0,0 @@ -package org.hisp.dhis.fileresource; - -/* - * Copyright (c) 2004-2015, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import java.io.IOException; -import java.io.InputStream; -import java.util.Optional; -import java.util.Properties; - -import org.apache.commons.io.input.NullInputStream; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.domain.Credentials; -import org.jclouds.domain.Location; - -import com.google.common.hash.HashCode; -import com.google.common.io.ByteSource; - -/** - * @author Halvdan Hoem Grelland - */ -public abstract class BaseJCloudsFileResourceContentStore - implements FileResourceContentStore -{ - private BlobStore blobStore; - private BlobStoreContext blobStoreContext; - - // ------------------------------------------------------------------------- - // Default config implementations - // ------------------------------------------------------------------------- - - protected Credentials getCredentials() - { - return new Credentials( "Unused", "Unused" ); - } - - protected Properties getOverrides() - { - return new Properties(); - } - - // ------------------------------------------------------------------------- - // Abstract methods - // ------------------------------------------------------------------------- - - protected abstract String getContainer(); - - protected abstract String getJCloudsProviderKey(); - - protected abstract String getLocation(); - - // ------------------------------------------------------------------------- - // Life cycle management - // ------------------------------------------------------------------------- - - public void init() - { - Credentials credentials = getCredentials(); - Properties overrides = getOverrides(); - String providerKey = getJCloudsProviderKey(); - - blobStoreContext = ContextBuilder.newBuilder( providerKey ) - .credentials( credentials.identity, credentials.credential ) - .overrides( overrides ).build( BlobStoreContext.class ); - - blobStore = blobStoreContext.getBlobStore(); - - Optional location = blobStore.listAssignableLocations() - .stream().filter( l -> l.getId().equals( getLocation() ) ).findFirst(); - - blobStore.createContainerInLocation( location.isPresent() ? location.get() : null, getContainer() ); - } - - public void cleanUp() - { - blobStoreContext.close(); - } - - // ------------------------------------------------------------------------- - // FileResourceContentStore implementation - // ------------------------------------------------------------------------- - - public ByteSource getFileResourceContent( String key ) - { - final Blob blob = getBlob( key ); - - if ( blob == null ) - { - return null; - } - - return new ByteSource() - { - @Override - public InputStream openStream() - { - try - { - return blob.getPayload().openStream(); - } - catch ( IOException e ) - { - return new NullInputStream( 0 ); - } - } - }; - } - - public String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 ) - { - Blob blob = createBlob( key, content, size, contentMd5 ); - - if ( blob == null ) - { - return null; - } - - putBlob( blob ); - - return key; - } - - public void deleteFileResourceContent( String key ) - { - deleteBlob( key ); - } - - // ------------------------------------------------------------------------- - // Supportive methods - // ------------------------------------------------------------------------- - - private Blob getBlob( String key ) - { - return blobStore.getBlob( getContainer(), key ); - } - - private void deleteBlob( String key ) - { - blobStore.removeBlob( getContainer(), key ); - } - - private String putBlob( Blob blob ) - { - return blobStore.putBlob( getContainer(), blob ); - } - - private Blob createBlob( String key, ByteSource content, long size, String contentMd5 ) - { - return blobStore.blobBuilder( key ) - .payload( content ) - .contentLength( size ) - .contentMD5( HashCode.fromString( contentMd5 ) ) - .build(); - } -} === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 2015-09-18 17:50:30 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceContentStore.java 2015-09-18 21:24:50 +0000 @@ -28,39 +28,57 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import com.google.common.hash.HashCode; +import com.google.common.io.ByteSource; +import org.apache.commons.io.input.NullInputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.external.location.LocationManager; import org.hisp.dhis.hibernate.HibernateConfigurationProvider; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; import org.jclouds.domain.Credentials; +import org.jclouds.domain.Location; import org.jclouds.filesystem.reference.FilesystemConstants; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.stream.Collectors; /** - * TODO Merge with BaseJCloudsFileResourceContentStore ? * @author Halvdan Hoem Grelland */ public class DefaultFileResourceContentStore - extends BaseJCloudsFileResourceContentStore + implements FileResourceContentStore { private static final Log log = LogFactory.getLog( DefaultFileResourceContentStore.class ); + private BlobStore blobStore; + private BlobStoreContext blobStoreContext; + private String container; + // ------------------------------------------------------------------------- - // Provider constants + // Providers // ------------------------------------------------------------------------- private static final String JCLOUDS_PROVIDER_KEY_FILESYSTEM = "filesystem"; private static final String JCLOUDS_PROVIDER_KEY_AWS_S3 = "aws-s3"; + private static final String JCLOUDS_PROVIDER_KEY_TRANSIENT = "transient"; - private static final List AVAILABLE_PROVIDERS = new ArrayList() {{ - addAll( Arrays.asList( JCLOUDS_PROVIDER_KEY_FILESYSTEM, JCLOUDS_PROVIDER_KEY_AWS_S3 ) ); + private static final List SUPPORTED_PROVIDERS = new ArrayList() {{ + addAll( Arrays.asList( + JCLOUDS_PROVIDER_KEY_FILESYSTEM, + JCLOUDS_PROVIDER_KEY_AWS_S3 + ) ); }}; // ------------------------------------------------------------------------- @@ -79,22 +97,9 @@ // Defaults // ------------------------------------------------------------------------- - private static final String DEFAULT_PROVIDER = "filesystem"; private static final String DEFAULT_CONTAINER = "dhis2_filestore"; // ------------------------------------------------------------------------- - // Configuration - // ------------------------------------------------------------------------- - - private Map filestoreConfiguration; - - private String provider; - private String container; - private Credentials credentials; - private String location; - private Properties overrides = new Properties(); - - // ------------------------------------------------------------------------- // Dependencies // ------------------------------------------------------------------------- @@ -113,115 +118,163 @@ } // ------------------------------------------------------------------------- - // Lifecycle + // Life cycle management // ------------------------------------------------------------------------- + // TODO Untangle and split up public void init() { + // ------------------------------------------------------------------------- + // Parse properties + // ------------------------------------------------------------------------- + Properties properties = configurationProvider.getConfiguration().getProperties(); - - filestoreConfiguration = properties - .entrySet().stream().filter( - p -> ( (String) p.getKey() ).startsWith( FILESTORE_CONFIG_NAMESPACE ) ) + + Map filestoreConfiguration = properties + .entrySet().stream().filter( p -> ((String) p.getKey()).startsWith( FILESTORE_CONFIG_NAMESPACE ) ) .collect( Collectors.toMap( p -> StringUtils.strip( (String) p.getKey() ), p -> StringUtils.strip( (String) p.getValue() ) ) ); - provider = filestoreConfiguration.getOrDefault( KEY_FILESTORE_PROVIDER, DEFAULT_PROVIDER ); - - if ( !AVAILABLE_PROVIDERS.contains( provider ) ) - { - log.info( "Ignored unsupported file store provider '" + provider + "', falling back to file system." ); - provider = DEFAULT_PROVIDER; + String provider = filestoreConfiguration.getOrDefault( KEY_FILESTORE_PROVIDER, JCLOUDS_PROVIDER_KEY_FILESYSTEM ); + + if ( !SUPPORTED_PROVIDERS.contains( provider ) ) + { + log.warn( "Ignored unsupported file store provider '" + provider + "', falling back to file system." ); + provider = JCLOUDS_PROVIDER_KEY_FILESYSTEM; + } + + if ( provider.equals( JCLOUDS_PROVIDER_KEY_FILESYSTEM ) && !locationManager.externalDirectorySet() ) + { + log.warn( "File system store provider configured but external directory is not set. Falling back to transient file store." ); + provider = JCLOUDS_PROVIDER_KEY_TRANSIENT; } container = filestoreConfiguration.getOrDefault( KEY_FILESTORE_CONTAINER, DEFAULT_CONTAINER ); - location = filestoreConfiguration.getOrDefault( KEY_FILESTORE_LOCATION, null ); - - switch ( provider ) - { - case JCLOUDS_PROVIDER_KEY_FILESYSTEM: - configureFilesystemProvider(); - break; - case JCLOUDS_PROVIDER_KEY_AWS_S3: - configureAWSS3Provider(); - break; - default: - throw new IllegalArgumentException( "The filestore provider " + provider + " is not supported." ); - } - - super.init(); + String location = filestoreConfiguration.getOrDefault( KEY_FILESTORE_LOCATION, null ); + Properties overrides = new Properties(); + Credentials credentials = new Credentials( "Unused", "Unused" ); + + // ------------------------------------------------------------------------- + // Provider specific configuration + // ------------------------------------------------------------------------- + + if ( provider.equals( JCLOUDS_PROVIDER_KEY_FILESYSTEM ) && locationManager.externalDirectorySet() ) + { + overrides.setProperty( FilesystemConstants.PROPERTY_BASEDIR, locationManager.getExternalDirectoryPath() ); + + log.info( "File system store provider configured" ); + } + else if ( provider.equals( JCLOUDS_PROVIDER_KEY_AWS_S3 ) ) + { + credentials = new Credentials( filestoreConfiguration.getOrDefault( + KEY_FILESTORE_IDENTITY, "" ), filestoreConfiguration.getOrDefault( KEY_FILESTORE_SECRET, "" ) ); + + log.info( "AWS S3 filestore provider configured." ); + + if ( credentials.identity.isEmpty() || credentials.credential.isEmpty() ) + { + log.info( "AWS S3 store configured with empty credentials, authentication not possible" ); + } + } + + // ------------------------------------------------------------------------- + // Set up BlobStore + // ------------------------------------------------------------------------- + + blobStoreContext = ContextBuilder.newBuilder( provider ) + .credentials( credentials.identity, credentials.credential ) + .overrides( overrides ).build( BlobStoreContext.class ); + + blobStore = blobStoreContext.getBlobStore(); + + Optional configuredLocation = blobStore.listAssignableLocations() + .stream().filter( l -> l.getId().equals( location ) ).findFirst(); + + blobStore.createContainerInLocation( configuredLocation.isPresent() ? configuredLocation.get() : null, container ); } public void cleanUp() { - super.cleanUp(); - } - - // ------------------------------------------------------------------------- - // Configuration implementation - // ------------------------------------------------------------------------- - - @Override - protected Properties getOverrides() - { - return overrides; - } - - @Override - protected Credentials getCredentials() - { - return credentials; - } - - @Override - protected String getContainer() - { - return container; - } - - @Override - protected String getLocation() - { - return location; - } - - @Override - protected String getJCloudsProviderKey() - { - return provider; + blobStoreContext.close(); + } + + // ------------------------------------------------------------------------- + // FileResourceContentStore implementation + // ------------------------------------------------------------------------- + + public ByteSource getFileResourceContent( String key ) + { + final Blob blob = getBlob( key ); + + if ( blob == null ) + { + return null; + } + + return new ByteSource() + { + @Override + public InputStream openStream() + { + try + { + return blob.getPayload().openStream(); + } + catch ( IOException e ) + { + return new NullInputStream( 0 ); + } + } + }; + } + + public String saveFileResourceContent( String key, ByteSource content, long size, String contentMd5 ) + { + Blob blob = createBlob( key, content, size, contentMd5 ); + + if ( blob == null ) + { + return null; + } + + putBlob( blob ); + + return key; + } + + public void deleteFileResourceContent( String key ) + { + deleteBlob( key ); } // ------------------------------------------------------------------------- // Supportive methods // ------------------------------------------------------------------------- - private void configureFilesystemProvider() - { - if ( locationManager.externalDirectorySet() ) - { - overrides.setProperty( FilesystemConstants.PROPERTY_BASEDIR, locationManager.getExternalDirectoryPath() ); - credentials = super.getCredentials(); - - log.info( "File system store provider configured" ); - } - else - { - log.warn( "File system store could not be configured, external directory not set" ); - } - } - - private void configureAWSS3Provider() - { - credentials = new Credentials( filestoreConfiguration.getOrDefault( - KEY_FILESTORE_IDENTITY, "" ), filestoreConfiguration.getOrDefault( KEY_FILESTORE_SECRET, "" ) ); - log.info( "AWS S3 filestore provider configured." ); - - if ( credentials.identity.isEmpty() || credentials.credential.isEmpty() ) - { - log.info( "AWS S3 store configured with empty credentials, authentication not possible" ); - } + private Blob getBlob( String key ) + { + return blobStore.getBlob( container, key ); + } + + private void deleteBlob( String key ) + { + blobStore.removeBlob( container, key ); + } + + private String putBlob( Blob blob ) + { + return blobStore.putBlob( container, blob ); + } + + private Blob createBlob( String key, ByteSource content, long size, String contentMd5 ) + { + return blobStore.blobBuilder( key ) + .payload( content ) + .contentLength( size ) + .contentMD5( HashCode.fromString( contentMd5 ) ) + .build(); } } === modified file 'dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java' --- dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-09-18 20:02:57 +0000 +++ dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java 2015-09-18 21:24:50 +0000 @@ -33,6 +33,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hisp.dhis.common.GenericIdentifiableObjectStore; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; import org.springframework.transaction.annotation.Transactional; /** @@ -41,9 +43,12 @@ public class DefaultFileResourceService implements FileResourceService { - private static final Log log = LogFactory.getLog( DefaultFileResourceService.class ); + private BlobStore blobStore; + + private BlobStoreContext blobStoreContext; + // ------------------------------------------------------------------------- // Dependencies // -------------------------------------------------------------------------