S3UploadPerfTest.java

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

import com.amazonaws.HttpMethod;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.eaio.uuid.UUID;

/**
 * Created by boxu on 7/20/16.
 */
public class S3UploadPerfTest {

    private static String bucketName = "txb-perf-test";
    protected AmazonS3 s3client;
    protected TransferManager transferManager;
    private final int times = 50; 
    private final String uploadFileName = "/Users/boxu/Desktop/t-normal.json";

    @Before
    public void testSetup() throws Exception {
        s3client = new AmazonS3Client(new ProfileCredentialsProvider());
        transferManager = new TransferManager(new ProfileCredentialsProvider());
    }

    @Test
    public void testPublishUsingPutObject() throws Exception {
        String path = "/putObject/large/";
        String keyName;
        long sum = 0;
        for (int i = 0; i < times; ++i) {
            keyName = path + new UUID().toString();
//            System.out.println("Uploading a new object to S3 from a file\n");
            File file = new File(uploadFileName);
            long startTime = System.currentTimeMillis();
            s3client.putObject(new PutObjectRequest(
                    bucketName, keyName, file));
            long endTime = System.currentTimeMillis();
            long totalTime = endTime - startTime;
            sum += totalTime;
        }
        purgeBucket(path);

        System.out.println(sum / times);
        assertTrue(true);
    }

    @Test
    public void testPublishUsingMultipart() throws Exception {
        String path = "/multipart/large/";
        String keyName;
        File file = new File(uploadFileName);
        long sum = 0;
        for (int i = 0; i < times; ++i) {
            keyName = path + new UUID().toString();
//            System.out.println("Uploading a new object to S3 from a file\n");
            long startTime = System.currentTimeMillis();
            Upload upload = transferManager.upload(new PutObjectRequest(
                    bucketName, keyName, file));
            upload.waitForCompletion();
            long endTime = System.currentTimeMillis();
            long totalTime = endTime - startTime;
            sum += totalTime;
        }
        purgeBucket(path);

        System.out.println(sum / times);
        assertTrue(true);
    }

    @Test
    public void testPublishUsingPresignedURL() throws Exception {
        String path = "/presignedURL/normal/";
        String keyName;
        File file = new File(uploadFileName);
        long sum = 0;
        for (int i = 0; i < times; ++i) {
            keyName = path + new UUID().toString();
//            System.out.println("Generating pre-signed URL.");

            java.util.Date expiration = new java.util.Date();
            long milliSeconds = expiration.getTime();
            milliSeconds += 1000 * 60 * 10; // Add 10 minutes.
            expiration.setTime(milliSeconds);

            long startTime = System.currentTimeMillis();
            GeneratePresignedUrlRequest generatePresignedUrlRequest =
                    new GeneratePresignedUrlRequest(bucketName, keyName);
            generatePresignedUrlRequest.setMethod(HttpMethod.PUT);
            generatePresignedUrlRequest.setExpiration(expiration);

            URL url = s3client.generatePresignedUrl(generatePresignedUrlRequest);

            UploadObject(url, file);
            long endTime = System.currentTimeMillis();
            long totalTime = endTime - startTime;
            sum += totalTime;
        }
        purgeBucket(path);

        System.out.println(sum / times);
        assertTrue(true);
    }

    public static void UploadObject(URL url, File file) throws IOException
    {
        HttpURLConnection connection=(HttpURLConnection) url.openConnection();
        connection.setDoOutput(true);
        connection.setRequestMethod("PUT");

        IOUtils.copy(new FileInputStream(file), connection.getOutputStream());
        int responseCode = connection.getResponseCode();

    }

    private void purgeBucket(String folder) throws Exception {
        List<String> objectKeys = getObjectKeys(folder);
        System.out.println("delete files: " + objectKeys.size());
        DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName);
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>();
        for (String objectKey : objectKeys) {
            keys.add(new DeleteObjectsRequest.KeyVersion(objectKey));
        }
        if (keys.isEmpty()) {
            return;
        }
        multiObjectDeleteRequest.setKeys(keys);
        s3client.deleteObjects(multiObjectDeleteRequest);
    }

    private List<String> getObjectKeys(String prefix) throws Exception {
        List<String> keys = new ArrayList<>();
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix(prefix);
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                keys.add(objectSummary.getKey());
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
        return keys;
    }
}