NativeS3FileSystem input test

1GB

# condition
S3
 - Bucket region: Tokyo
 - Size: 1GB
EMR
 - AZ: ap-northeast-1a
 - Type:m1.lage
 - hadoop-version: 0.20.205
 - ami-version: 2.0.2 (latest)

# command
hadoop jar fs-input-perfcheck.jar com.example.FsInputPerfCheck s3://***/input.csv

# output
path = s3://***/input.csv
fs = org.apache.hadoop.fs.s3native.NativeS3FileSystem
size = 1073741854
12/01/25 02:16:48 INFO s3native.NativeS3FileSystem: Opening 's3://***/input.csv' for reading
stream open = 113(ms)
12/01/25 02:17:22 INFO s3native.NativeS3FileSystem: Stream for key '***/input.csv' seeking to position '1'
stream seek = 33937(ms)
stream close = 33272(ms)

1MB

# condition
S3
 - Bucket region: Tokyo
 - Size: 1MB
EMR
 - AZ: ap-northeast-1a
 - Type:m1.lage
 - hadoop-version: 0.20.205
 - ami-version: 2.0.2 (latest)

# command
hadoop jar fs-input-perfcheck.jar com.example.FsInputPerfCheck s3://***/input1.csv

# output
path = s3://***/input1.csv
fs = org.apache.hadoop.fs.s3native.NativeS3FileSystem
size = 1024587
12/01/25 02:16:02 INFO s3native.NativeS3FileSystem: Opening 's3://***/input1.csv' for reading
stream open = 116(ms)
12/01/25 02:16:03 INFO s3native.NativeS3FileSystem: Stream for key '***/input1.csv' seeking to position '1'
stream seek = 208(ms)
stream close = 32(ms)

FsInputPerfCheck.java

package com.example;

import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class FsInputPerfCheck extends Configured implements Tool {

    @Override
    public int run(String[] args) throws Exception {
        if (args.length != 1) {
            System.err.printf("Usage: hadoop jar <this-jar> target-uri %s%n", getClass().getName());
            throw new IllegalArgumentException(Arrays.toString(args));
        }
        Path path = new Path(args[0]);
        System.err.printf("path = %s%n", path);

        FileSystem fs = path.getFileSystem(getConf());
        System.out.printf("fs = %s%n", fs.getClass().getName());

        FileStatus stat = fs.getFileStatus(path);
        System.out.printf("size = %s%n", stat.getLen());

        long start = System.currentTimeMillis();
        long opened, sought;
        FSDataInputStream input = fs.open(path);
        try {
            opened = System.currentTimeMillis();
            System.out.printf("stream open = %s(ms)%n", opened - start);

            input.seek(1);
            sought = System.currentTimeMillis();
            System.out.printf("stream seek = %s(ms)%n", sought - opened);
        } finally {
            input.close();
        }
        long finish = System.currentTimeMillis();
        System.out.printf("stream close = %s(ms)%n", finish - sought);
        return 0;
    }

    public static void main(String[] args) throws Exception {
        int exitCode = ToolRunner.run(new Configuration(), new FsInputPerfCheck(), args);
        System.exit(exitCode);
    }
}