Azure Storage Client 2.0, Simple Table Storage performance measurement

Program.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Table;

namespace ConsoleApplication13
{

    class Program
    {
        async static Task<Tuple<long,long>> Insert(CloudTable table, long n)
        {
            var partitionKey = string.Format("{0:D12}", n % 100);
            var e = new EntityNk(partitionKey, Guid.NewGuid().ToString() + "-" + n.ToString("D12"));
            var tableOperation = TableOperation.Insert(e);

            var sw = Stopwatch.StartNew();
            var result = await table.ExecuteAsync(tableOperation);

            return new Tuple<long,long>(DateTime.UtcNow.Ticks, sw.ElapsedTicks);
        }

        static IList<Tuple<double, double>> Run(CloudTable table)
        {
            var start = DateTime.UtcNow.Ticks;

            var tasks = Enumerable.Range(0, 1000 * 200).Select(n =>
            {
                var e = Insert(table, n);
                return e;

            }).ToArray();

            Task.WaitAll(tasks.ToArray());

            var result = tasks.Select(t => new Tuple<double, double>(
                (double)(t.Result.Item1 - start) / TimeSpan.TicksPerMillisecond,
                (double)t.Result.Item2 / TimeSpan.TicksPerMillisecond)).ToArray();

            return result;
        }
        
        static void Main(string[] args)
        {
            ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 12 * 4 * 10;
            ServicePointManager.Expect100Continue = false;
            ServicePointManager.UseNagleAlgorithm = false;

            var storageAccount = args.Length > 0 ? CloudStorageAccount.Parse(args[0]) : CloudStorageAccount.DevelopmentStorageAccount;

            var tableClient = storageAccount.CreateCloudTableClient().GetTableReference("TestTable"+Guid.NewGuid().ToString("N"));

            tableClient.CreateIfNotExists();

            for (var i = 1; i < 64; i *= 2)
            {
                EntityNk.DataSize = i;

                var result = Run(tableClient);
                Console.WriteLine("# Data Size {0}K", i);
                Console.WriteLine("# ElapsedTime(ms) ExecutionTime(ms)");

                // 全件
                foreach (var t in result.OrderBy(t => t.Item1))
                    Console.WriteLine("{0} {1}", t.Item1, t.Item2);

                Console.Write("\n\n");
                
                // ms で四捨五入して
                Console.WriteLine("# ExecutionTime(ms) NumberOfCounts");
                foreach (var t in result.GroupBy(t => Math.Round(t.Item2)).OrderBy(t => t.Key))
                    Console.WriteLine("{0} {1}", t.Key, t.Count());

                Console.Write("\n\n");
            }
        }
    }

    public class EntityNk : TableEntity
    {
        public const string DATA_1K = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdef";
        public static string DataCache = DATA_1K;
        private static int dataSize = 1;


        public static int DataSize
        {
            set
            {
                if (value != dataSize)
                {
                    var sb = new StringBuilder(value);
                    for (var i = 0; i < value; i++)
                    {
                        sb.Append(DATA_1K);
                    }
                    DataCache = sb.ToString();
                    dataSize = value;
                }
            }
        }


        public EntityNk(string partitionKey, string rowKey)
        {
            this.PartitionKey = partitionKey;
            this.RowKey = rowKey;
            this.Data = DataCache; // save memory
        }

        public EntityNk() { }

        public string Data { get; set; }

    }

    public static class CloudTableExtensions
    {
        public static Task<TableResult> ExecuteAsync(this CloudTable cloudTable, TableOperation operation, TableRequestOptions requestOptions = null, OperationContext operationContext = null, object state = null)
        {
            return Task.Factory.FromAsync<TableOperation, TableRequestOptions, OperationContext, TableResult>(
                cloudTable.BeginExecute, cloudTable.EndExecute, operation, requestOptions, operationContext, state);
        }
    }
}

説明

コマンドラインで、ストレージへの接続文字列を渡して実行します。なにも渡さないと開発ストレージに接続しようとします。開発ストレージには、SDK 1.8のものが必要です。

実行すると、TableTest＋Guildのテーブルを作成し、1K、2K、4K、8K、16K、32Kのエンティティを各200K行Insertします。データはパーテーションは100分割され、各パーテーションには、各サイズのエンティティが2000づつ合計12000行入ります。

結果は標準出力に出ます。結果は最初のブロックが各エンティティのInsertにかかった時間、次のブロックがms単位で丸めた結果をGroup By Countしたものが、１Kから３２Kまで繰り返します。

結果をファイルに落してGnuplotで見ると全体像がわかりやすいようです。

$ ConsoleApplication13 "ConnectionString" > n10kd0164p100.dat


plot "n10kd0164p100.dat" index 1 title '1K' with linespoints smooth csplines, \
  "n10kd0164p100.dat" index 3 title '2K' with linespoints smooth csplines, \
  "n10kd0164p100.dat" index 5 title '4K' with linespoints smooth csplines, \
  "n10kd0164p100.dat" index 7 title '8K' with linespoints smooth csplines, \
  "n10kd0164p100.dat" index 9 title '16K' with linespoints smooth csplines, \
  "n10kd0164p100.dat" index 11 title '32K' with linespoints smooth csplines