gh-giuseppe-trisciuoglio-developer-kit-developer-kit/skills/aws-sdk-java-v2-s3/references/s3-spring-boot-integration.md

S3 Spring Boot Integration Reference

Advanced Spring Boot Configuration

Multi-Environment Configuration

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;

@Configuration
@EnableConfigurationProperties(S3Properties.class)
public class S3Configuration {

    private final S3Properties properties;

    public S3Configuration(S3Properties properties) {
        this.properties = properties;
    }

    @Bean
    @ConditionalOnProperty(name = "s3.client.async.enabled", havingValue = "true")
    public S3AsyncClient s3AsyncClient() {
        return S3AsyncClient.builder()
            .region(Region.of(properties.getRegion()))
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    properties.getAccessKey(),
                    properties.getSecretKey())))
            .endpointOverride(URI.create(properties.getEndpoint()))
            .build();
    }

    @Bean
    @ConditionalOnProperty(name = "s3.client.sync.enabled", havingValue = "true", matchIfMissing = true)
    public S3Client s3Client() {
        return S3Client.builder()
            .region(Region.of(properties.getRegion()))
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    properties.getAccessKey(),
                    properties.getSecretKey())))
            .endpointOverride(URI.create(properties.getEndpoint()))
            .build();
    }

    @Bean
    @ConditionalOnProperty(name = "s3.transfer-manager.enabled", havingValue = "true")
    public S3TransferManager s3TransferManager() {
        return S3TransferManager.builder()
            .s3Client(s3Client())
            .build();
    }

    @Bean
    @ConditionalOnProperty(name = "s3.presigner.enabled", havingValue = "true")
    public S3Presigner s3Presigner() {
        return S3Presigner.builder()
            .region(Region.of(properties.getRegion()))
            .build();
    }
}

@ConfigurationProperties(prefix = "s3")
@Data
public class S3Properties {
    private String accessKey;
    private String secretKey;
    private String region = "us-east-1";
    private String endpoint = null;
    private boolean syncEnabled = true;
    private boolean asyncEnabled = false;
    private boolean transferManagerEnabled = false;
    private boolean presignerEnabled = false;
    private int maxConnections = 100;
    private int connectionTimeout = 5000;
    private int socketTimeout = 30000;
    private String defaultBucket;
}

Profile-Specific Configuration

# application-dev.properties
s3.access-key=${AWS_ACCESS_KEY}
s3.secret-key=${AWS_SECRET_KEY}
s3.region=us-east-1
s3.endpoint=http://localhost:4566
s3.async-enabled=true
s3.transfer-manager-enabled=true

# application-prod.properties
s3.access-key=${AWS_ACCESS_KEY}
s3.secret-key=${AWS_SECRET_KEY}
s3.region=us-east-1
s3.async-enabled=true
s3.presigner-enabled=true

Advanced Service Patterns

Generic S3 Service Template

import software.amazon.awssdk.services.s3.model.*;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.nio.file.*;
import java.util.*;
import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
public class S3Service {

    private final S3Client s3Client;
    private final S3AsyncClient s3AsyncClient;
    private final S3TransferManager transferManager;
    private final S3Properties s3Properties;

    // Basic Operations
    public Mono<Void> uploadObjectAsync(String key, byte[] data) {
        return Mono.fromFuture(() -> {
            PutObjectRequest request = PutObjectRequest.builder()
                .bucket(s3Properties.getDefaultBucket())
                .key(key)
                .build();

            return s3AsyncClient.putObject(request,
                RequestBody.fromBytes(data)).future();
        });
    }

    public Mono<byte[]> downloadObjectAsync(String key) {
        return Mono.fromFuture(() -> {
            GetObjectRequest request = GetObjectRequest.builder()
                .bucket(s3Properties.getDefaultBucket())
                .key(key)
                .build();

            return s3AsyncClient.getObject(request)
                .thenApply(response -> {
                    try {
                        return response.readAllBytes();
                    } catch (IOException e) {
                        throw new RuntimeException("Failed to read S3 object", e);
                    }
                });
        });
    }

    // Advanced Operations
    public Mono<UploadResult> uploadWithMetadata(String key,
                                                  Path file,
                                                  Map<String, String> metadata) {
        return Mono.fromFuture(() -> {
            PutObjectRequest request = PutObjectRequest.builder()
                .bucket(s3Properties.getDefaultBucket())
                .key(key)
                .metadata(metadata)
                .contentType(getContentType(file))
                .build();

            return s3AsyncClient.putObject(request, RequestBody.fromFile(file))
                .thenApply(response -> new UploadResult(key, response.eTag()));
        });
    }

    public Flux<S3Object> listObjectsWithPrefix(String prefix) {
        ListObjectsV2Request request = ListObjectsV2Request.builder()
            .bucket(s3Properties.getDefaultBucket())
            .prefix(prefix)
            .build();

        return Flux.create(sink -> {
            s3Client.listObjectsV2Paginator(request)
                .contents()
                .forEach(sink::next);
            sink.complete();
        });
    }

    public Mono<Void> batchDelete(List<String> keys) {
        return Mono.fromFuture(() -> {
            List<ObjectIdentifier> objectIdentifiers = keys.stream()
                .map(key -> ObjectIdentifier.builder().key(key).build())
                .collect(Collectors.toList());

            Delete delete = Delete.builder()
                .objects(objectIdentifiers)
                .build();

            DeleteObjectsRequest request = DeleteObjectsRequest.builder()
                .bucket(s3Properties.getDefaultBucket())
                .delete(delete)
                .build();

            return s3AsyncClient.deleteObjects(request).future();
        });
    }

    // Transfer Manager Operations
    public Mono<UploadResult> uploadWithTransferManager(String key, Path file) {
        return Mono.fromFuture(() -> {
            UploadFileRequest request = UploadFileRequest.builder()
                .putObjectRequest(req -> req
                    .bucket(s3Properties.getDefaultBucket())
                    .key(key))
                .source(file)
                .build();

            return transferManager.uploadFile(request)
                .completionFuture()
                .thenApply(result -> new UploadResult(key, result.response().eTag()));
        });
    }

    public Mono<DownloadResult> downloadWithTransferManager(String key, Path destination) {
        return Mono.fromFuture(() -> {
            DownloadFileRequest request = DownloadFileRequest.builder()
                .getObjectRequest(req -> req
                    .bucket(s3Properties.getDefaultBucket())
                    .key(key))
                .destination(destination)
                .build();

            return transferManager.downloadFile(request)
                .completionFuture()
                .thenApply(result -> new DownloadResult(destination, result.response().contentLength()));
        });
    }

    // Utility Methods
    private String getContentType(Path file) {
        try {
            return Files.probeContentType(file);
        } catch (IOException e) {
            return "application/octet-stream";
        }
    }

    // Records for Results
    public record UploadResult(String key, String eTag) {}
    public record DownloadResult(Path path, long size) {}
}

Event-Driven S3 Operations

import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import reactor.core.publisher.Mono;

@Service
@RequiredArgsConstructor
public class S3EventService {

    private final S3Service s3Service;
    private final ApplicationEventPublisher eventPublisher;

    @Transactional
    public Mono<UploadResult> uploadAndPublishEvent(String key, Path file) {
        return s3Service.uploadWithTransferManager(key, file)
            .doOnSuccess(result -> {
                eventPublisher.publishEvent(new S3UploadEvent(key, result.eTag()));
            })
            .doOnError(error -> {
                eventPublisher.publishEvent(new S3UploadFailedEvent(key, error.getMessage()));
            });
    }

    public Mono<String> generatePresignedUrl(String key) {
        return s3Service.downloadObjectAsync(key)
            .flatMap(data -> {
                return Mono.fromCallable(() -> {
                    S3Presigner presigner = S3Presigner.create();
                    try {
                        GetObjectRequest request = GetObjectRequest.builder()
                            .bucket(s3Service.getDefaultBucket())
                            .key(key)
                            .build();

                        GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
                            .signatureDuration(Duration.ofMinutes(10))
                            .getObjectRequest(request)
                            .build();

                        return presigner.presignGetObject(presignRequest)
                            .url()
                            .toString();
                    } finally {
                        presigner.close();
                    }
                });
            });
    }
}

// Event Classes
public class S3UploadEvent extends ApplicationEvent {
    private final String key;
    private final String eTag;

    public S3UploadEvent(String key, String eTag) {
        super(key);
        this.key = key;
        this.eTag = eTag;
    }

    public String getKey() { return key; }
    public String getETag() { return eTag; }
}

public class S3UploadFailedEvent extends ApplicationEvent {
    private final String key;
    private final String errorMessage;

    public S3UploadFailedEvent(String key, String errorMessage) {
        super(key);
        this.key = key;
        this.errorMessage = errorMessage;
    }

    public String getKey() { return key; }
    public String getErrorMessage() { return errorMessage; }
}

Retry and Error Handling

import org.springframework.retry.annotation.*;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Mono;
import software.amazon.awssdk.services.s3.model.*;

@Service
@RequiredArgsConstructor
public class ResilientS3Service {

    private final S3Client s3Client;
    private final RetryTemplate retryTemplate;

    @Retryable(value = {S3Exception.class, SdkClientException.class},
               maxAttempts = 3,
               backoff = @Backoff(delay = 1000, multiplier = 2))
    public Mono<PutObjectResponse> uploadWithRetry(String key, Path file) {
        return Mono.fromCallable(() -> {
            PutObjectRequest request = PutObjectRequest.builder()
                .bucket("my-bucket")
                .key(key)
                .build();

            return s3Client.putObject(request, RequestBody.fromFile(file));
        });
    }

    @Recover
    public Mono<PutObjectResponse> uploadRecover(S3Exception e, String key, Path file) {
        // Log the failure and potentially send notification
        System.err.println("Upload failed after retries: " + e.getMessage());
        return Mono.error(new S3UploadException("Upload failed after retries", e));
    }

    @Retryable(value = {S3Exception.class},
               maxAttempts = 5,
               backoff = @Backoff(delay = 2000, multiplier = 2))
    public Mono<Void> copyObjectWithRetry(String sourceKey, String destinationKey) {
        return Mono.fromFuture(() -> {
            CopyObjectRequest request = CopyObjectRequest.builder()
                .sourceBucket("source-bucket")
                .sourceKey(sourceKey)
                .destinationBucket("destination-bucket")
                .destinationKey(destinationKey)
                .build();

            return s3AsyncClient.copyObject(request).future();
        });
    }
}

public class S3UploadException extends RuntimeException {
    public S3UploadException(String message, Throwable cause) {
        super(message, cause);
    }
}

Testing Integration

Test Configuration with LocalStack

import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.test.context.ActiveProfiles;
import org.testcontainers.utility.DockerImageName;

@Testcontainers
@ActiveProfiles("test")
@TestConfiguration
public class S3TestConfig {

    @Container
    static LocalStackContainer localstack = new LocalStackContainer(
        DockerImageName.parse("localstack/localstack:3.0"))
        .withServices(LocalStackContainer.Service.S3)
        .withEnv("DEFAULT_REGION", "us-east-1");

    @Bean
    public S3Client testS3Client() {
        return S3Client.builder()
            .region(Region.US_EAST_1)
            .endpointOverride(localstack.getEndpointOverride(LocalStackContainer.Service.S3))
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    localstack.getAccessKey(),
                    localstack.getSecretKey())))
            .build();
    }

    @Bean
    public S3AsyncClient testS3AsyncClient() {
        return S3AsyncClient.builder()
            .region(Region.US_EAST_1)
            .endpointOverride(localstack.getEndpointOverride(LocalStackContainer.Service.S3))
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    localstack.getAccessKey(),
                    localstack.getSecretKey())))
            .build();
    }
}

Unit Testing with Mocks

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import reactor.core.publisher.Mono;
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;

@ExtendWith(MockitoExtension.class)
class S3ServiceTest {

    @Mock
    private S3Client s3Client;

    @InjectMocks
    private S3Service s3Service;

    @Test
    void uploadObjectAsync_ShouldReturnUploadResult() {
        // Arrange
        String key = "test-key";
        byte[] data = "test-content".getBytes();
        String eTag = "12345";

        PutObjectResponse response = PutObjectResponse.builder()
            .eTag(eTag)
            .build();

        when(s3Client.putObject(any(PutObjectRequest.class), any()))
            .thenReturn(response);

        // Act
        Mono<UploadResult> result = s3Service.uploadObjectAsync(key, data);

        // Assert
        result.subscribe(uploadResult -> {
            assertEquals(key, uploadResult.key());
            assertEquals(eTag, uploadResult.eTag());
        });
    }

    @Test
    void listObjectsWithPrefix_ShouldReturnObjectList() {
        // Arrange
        String prefix = "documents/";
        S3Object object1 = S3Object.builder().key("documents/file1.txt").build();
        S3Object object2 = S3Object.builder().key("documents/file2.txt").build();

        ListObjectsV2Response response = ListObjectsV2Response.builder()
            .contents(object1, object2)
            .build();

        when(s3Client.listObjectsV2(any(ListObjectsV2Request.class)))
            .thenReturn(response);

        // Act
        Flux<S3Object> result = s3Service.listObjectsWithPrefix(prefix);

        // Assert
        result.collectList()
            .subscribe(objects -> {
                assertEquals(2, objects.size());
                assertTrue(objects.stream().allMatch(obj -> obj.key().startsWith(prefix)));
            });
    }
}

Integration Testing

import org.junit.jupiter.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import software.amazon.awssdk.services.s3.model.*;
import java.nio.file.*;
import java.util.Map;

@SpringBootTest
@ActiveProfiles("test")
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class S3IntegrationTest {

    @Autowired
    private S3Service s3Service;

    private static final String TEST_BUCKET = "test-bucket";
    private static final String TEST_FILE = "test-document.txt";

    @BeforeAll
    static void setup() throws Exception {
        // Create test file
        Files.write(Paths.get(TEST_FILE), "Test content".getBytes());
    }

    @Test
    @Order(1)
    void uploadFile_ShouldSucceed() {
        // Act & Assert
        s3Service.uploadWithMetadata(TEST_FILE, Paths.get(TEST_FILE),
            Map.of("author", "test", "type", "document"))
            .as(StepVerifier::create)
            .expectNextMatches(result ->
                result.key().equals(TEST_FILE) && result.eTag() != null)
            .verifyComplete();
    }

    @Test
    @Order(2)
    void downloadFile_ShouldReturnContent() {
        // Act & Assert
        s3Service.downloadObjectAsync(TEST_FILE)
            .as(StepVerifier::create)
            .expectNext("Test content".getBytes())
            .verifyComplete();
    }

    @Test
    @Order(3)
    void listObjects_ShouldReturnFiles() {
        // Act & Assert
        s3Service.listObjectsWithPrefix("")
            .as(StepVerifier::create)
            .expectNextCount(1)
            .verifyComplete();
    }

    @AfterAll
    static void cleanup() {
        try {
            Files.deleteIfExists(Paths.get(TEST_FILE));
        } catch (IOException e) {
            // Ignore
        }
    }
}

Advanced Configuration Patterns

Environment-Specific Configuration

import org.springframework.boot.autoconfigure.condition.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import software.amazon.awssdk.auth.credentials.*;

@Configuration
public class EnvironmentAwareS3Config {

    @Bean
    @ConditionalOnMissingBean
    public AwsCredentialsProvider awsCredentialsProvider(S3Properties properties) {
        if (properties.getAccessKey() != null && properties.getSecretKey() != null) {
            return StaticCredentialsProvider.create(
                AwsBasicCredentials.create(
                    properties.getAccessKey(),
                    properties.getSecretKey()));
        }
        return DefaultCredentialsProvider.create();
    }

    @Bean
    @ConditionalOnMissingBean
    @ConditionalOnProperty(name = "s3.region")
    public Region region(S3Properties properties) {
        return Region.of(properties.getRegion());
    }

    @Bean
    @ConditionalOnMissingBean
    @ConditionalOnProperty(name = "s3.endpoint")
    public String endpoint(S3Properties properties) {
        return properties.getEndpoint();
    }
}

Multi-Bucket Support

import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.Map;

@Service
@RequiredArgsConstructor
public class MultiBucketS3Service {

    private final Map<String, S3Client> bucketClients = new HashMap<>();
    private final S3Client defaultS3Client;

    @Autowired
    public MultiBucketS3Service(S3Client defaultS3Client) {
        this.defaultS3Client = defaultS3Client;
    }

    public S3Client getClientForBucket(String bucketName) {
        return bucketClients.computeIfAbsent(bucketName, name ->
            S3Client.builder()
                .region(defaultS3Client.config().region())
                .credentialsProvider(defaultS3Client.config().credentialsProvider())
                .build());
    }

    public Mono<UploadResult> uploadToBucket(String bucketName, String key, Path file) {
        S3Client client = getClientForBucket(bucketName);
        // Upload implementation using the specific client
        return Mono.empty(); // Implementation
    }
}