Storage & backups

Java

3 min read

AWS SDK for Java 2.x is the official SDK for Java. The examples require Java 17 or later and SDK version 2.30 or later.

Installation

Add the dependencies to pom.xml (Maven):

<dependencyManagement>
  <dependencies>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>bom</artifactId>
      <version>2.31.0</version>
      <type>pom</type>
      <scope>import</scope>
    </dependency>
  </dependencies>
</dependencyManagement>

<dependencies>
  <dependency>
    <groupId>software.amazon.awssdk</groupId>
    <artifactId>s3</artifactId>
  </dependency>
  <dependency>
    <groupId>software.amazon.awssdk</groupId>
    <artifactId>s3-transfer-manager</artifactId>
  </dependency>
</dependencies>

For Gradle: implementation(platform("software.amazon.awssdk:bom:2.31.0")), implementation("software.amazon.awssdk:s3") and implementation("software.amazon.awssdk:s3-transfer-manager").

Access Keys

Pass the keys through environment variables — the SDK reads them automatically. Never store keys in the source code or in a repository.

export AWS_ACCESS_KEY_ID=<Access Key>
export AWS_SECRET_ACCESS_KEY=<Secret Key>

Create a Client

import java.net.URI;
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;

static final URI ENDPOINT = URI.create("https://s3.uzcstorage.uz");
static final Region REGION = Region.of("default");
static final String BUCKET = "my-bucket";

S3Client s3 = S3Client.builder()
        .region(REGION)
        .endpointOverride(ENDPOINT)
        .forcePathStyle(true)
        .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
        .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
        .build();

S3Client is thread-safe: create it once and reuse it in the whole application. The keys are read from the environment variables automatically.

Basic Operations

import java.nio.file.Paths;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.services.s3.model.*;

// Upload a file
s3.putObject(r -> r.bucket(BUCKET).key("docs/report.pdf").contentType("application/pdf"),
        RequestBody.fromFile(Paths.get("report.pdf")));

// Upload a string
s3.putObject(r -> r.bucket(BUCKET).key("docs/hello.txt"), RequestBody.fromString("Hello, UzCloud!"));

// Download to a file
s3.getObject(r -> r.bucket(BUCKET).key("docs/report.pdf"),
        ResponseTransformer.toFile(Paths.get("report-copy.pdf")));

// Read a small object into memory
String text = s3.getObjectAsBytes(r -> r.bucket(BUCKET).key("docs/hello.txt")).asUtf8String();

// Copy and delete
s3.copyObject(r -> r.sourceBucket(BUCKET).sourceKey("docs/report.pdf")
        .destinationBucket(BUCKET).destinationKey("archive/report.pdf"));
s3.deleteObject(r -> r.bucket(BUCKET).key("docs/report.pdf"));

List All Objects

One request returns at most 1,000 objects. To get all objects with a prefix, use a paginator:

s3.listObjectsV2Paginator(r -> r.bucket(BUCKET).prefix("docs/"))
        .contents()
        .forEach(o -> System.out.println(o.key() + " " + o.size()));

Large Files

S3 Transfer Manager uploads and downloads large files in parts, in parallel. It works on top of the asynchronous client:

import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.UploadFileRequest;
import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener;

S3AsyncClient asyncS3 = S3AsyncClient.builder()
        .region(REGION)
        .endpointOverride(ENDPOINT)
        .forcePathStyle(true)
        .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
        .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
        .multipartEnabled(true)
        .multipartConfiguration(c -> c.minimumPartSizeInBytes(64L * 1024 * 1024))
        .build();

try (S3TransferManager tm = S3TransferManager.builder().s3Client(asyncS3).build()) {
    tm.uploadFile(UploadFileRequest.builder()
                    .putObjectRequest(r -> r.bucket(BUCKET).key("backups/backup.tar.gz"))
                    .source(Paths.get("backup.tar.gz"))
                    .addTransferListener(LoggingTransferListener.create())
                    .build())
            .completionFuture()
            .join();
}

Streaming

To avoid loading a large object into memory, read and write it in chunks:

import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;

try (InputStream in = s3.getObject(r -> r.bucket(BUCKET).key("media/video.mp4"));
     OutputStream out = Files.newOutputStream(Paths.get("video-copy.mp4"))) {
    in.transferTo(out);
}

// Upload from a stream of known length (for example, an uploaded file in a web app)
try (InputStream in = Files.newInputStream(Paths.get("video.mp4"))) {
    long size = Files.size(Paths.get("video.mp4"));
    s3.putObject(r -> r.bucket(BUCKET).key("media/video.mp4"), RequestBody.fromInputStream(in, size));
}

Presigned URLs

A presigned URL gives temporary access to an object without sharing your keys. An upload link lets a browser or another service upload a file directly to the bucket — see Presigned URL and CORS.

import java.time.Duration;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;

try (S3Presigner presigner = S3Presigner.builder()
        .region(REGION)
        .endpointOverride(ENDPOINT)
        .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
        .build()) {

    // Download link, valid for 1 hour
    String getUrl = presigner.presignGetObject(p -> p
            .signatureDuration(Duration.ofHours(1))
            .getObjectRequest(r -> r.bucket(BUCKET).key("docs/report.pdf"))).url().toString();

    // Upload link, valid for 10 minutes
    String putUrl = presigner.presignPutObject(p -> p
            .signatureDuration(Duration.ofMinutes(10))
            .putObjectRequest(r -> r.bucket(BUCKET).key("uploads/photo.jpg").contentType("image/jpeg"))).url().toString();
}

Errors and Retries

The most common errors are a missing object (NoSuchKey, or 404 for HeadObject) and no access (AccessDenied, 403). Network errors and 5xx responses are retried automatically; the number of attempts is set when you create the client.

import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.S3Exception;

boolean objectExists(String key) {
    try {
        s3.headObject(r -> r.bucket(BUCKET).key(key));
        return true;
    } catch (NoSuchKeyException e) {
        return false;
    }
}

try {
    s3.getObject(r -> r.bucket(BUCKET).key("missing.txt"));
} catch (NoSuchKeyException e) {
    System.out.println("Object not found");
} catch (S3Exception e) {
    if (e.statusCode() == 403) System.out.println("Check the keys and the bucket policy");
    else throw e;
}

Spring Boot

Register the client as a bean and inject it where needed. Keep the endpoint and bucket in application.yml:

uzcloud:
  s3:
    endpoint: https://s3.uzcstorage.uz
    bucket: my-bucket
@Configuration
public class S3Config {
    @Bean
    S3Client s3Client(@Value("${uzcloud.s3.endpoint}") String endpoint) {
        return S3Client.builder()
                .region(Region.of("default"))
                .endpointOverride(URI.create(endpoint))
                .forcePathStyle(true)
                .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
                .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
                .build();
    }
}

Tip

See also: