Asynchronous Parallel Blob Transfers with Progress Change Notification 2.0

using System;
using System.ComponentModel;

using System.Collections.Generic;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using System.IO;
using System.Linq;

namespace BlobTransferUI
{

// Class to allow for easy async upload and download functions with progress change notifications
// Requires references to Microsoft.WindowsAzure.Storage.dll (Storage client 2.0) and Microsoft.WindowsAzure.StorageClient.dll (Storage client 1.7).
// See comments on UploadBlobAsync and DownloadBlobAsync functions for information on removing the 1.7 client library dependency
class BlobTransfer
    {
// Public async events
publicevent AsyncCompletedEventHandler TransferCompleted;
publicevent EventHandler<BlobTransferProgressChangedEventArgs> TransferProgressChanged;

// Public BlobTransfer properties
public TransferTypeEnum TransferType;

// Private variables
private ICancellableAsyncResult asyncresult;
privatebool Working = false;
privateobject WorkingLock = newobject();
private AsyncOperation asyncOp;

// Used to calculate download speeds
private Queue<long> timeQueue = new Queue<long>(200);
private Queue<long> bytesQueue = new Queue<long>(200);
private DateTime updateTime = System.DateTime.Now;

// Private BlobTransfer properties
privatestring m_FileName;
private ICloudBlob m_Blob;

// Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class.
// Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage).
publicvoid UploadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile)
        {
            Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey;
            ICloudBlob blob2 = new CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey()));
            UploadBlobAsync(blob2, LocalFile);
        }

// Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class.
// Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage).
publicvoid DownloadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile)
        {
            Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey;
            ICloudBlob blob2 = new CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey()));
            DownloadBlobAsync(blob2, LocalFile);
        }

publicvoid UploadBlobAsync(ICloudBlob blob, string LocalFile)
        {
// The class currently stores state in class level variables so calling UploadBlobAsync or DownloadBlobAsync a second time will cause problems.
// A better long term solution would be to better encapsulate the state, but the current solution works for the needs of my primary client.
// Throw an exception if UploadBlobAsync or DownloadBlobAsync has already been called.
lock (WorkingLock)
            {
if (!Working)
                    Working = true;
else
thrownew Exception("BlobTransfer already initiated. Create new BlobTransfer object to initiate a new file transfer.");
            }

// Attempt to open the file first so that we throw an exception before getting into the async work
using (FileStream fstemp = new FileStream(LocalFile, FileMode.Open, FileAccess.Read)) { }

// Create an async op in order to raise the events back to the client on the correct thread.
            asyncOp = AsyncOperationManager.CreateOperation(blob);

            TransferType = TransferTypeEnum.Upload;
            m_Blob = blob;
            m_FileName = LocalFile;

            var file = new FileInfo(m_FileName);
long fileSize = file.Length;

            FileStream fs = new FileStream(m_FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
            ProgressStream pstream = new ProgressStream(fs);
            pstream.ProgressChanged += pstream_ProgressChanged;
            pstream.SetLength(fileSize);
            m_Blob.ServiceClient.ParallelOperationThreadCount = 10;
            asyncresult = m_Blob.BeginUploadFromStream(pstream, BlobTransferCompletedCallback, new BlobTransferAsyncState(m_Blob, pstream));
        }

publicvoid DownloadBlobAsync(ICloudBlob blob, string LocalFile)
        {
// The class currently stores state in class level variables so calling UploadBlobAsync or DownloadBlobAsync a second time will cause problems.
// A better long term solution would be to better encapsulate the state, but the current solution works for the needs of my primary client.
// Throw an exception if UploadBlobAsync or DownloadBlobAsync has already been called.
lock (WorkingLock)
            {
if (!Working)
                    Working = true;
else
thrownew Exception("BlobTransfer already initiated. Create new BlobTransfer object to initiate a new file transfer.");
            }

// Create an async op in order to raise the events back to the client on the correct thread.
            asyncOp = AsyncOperationManager.CreateOperation(blob);

            TransferType = TransferTypeEnum.Download;
            m_Blob = blob;
            m_FileName = LocalFile;

            m_Blob.FetchAttributes();

            FileStream fs = new FileStream(m_FileName, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read);
            ProgressStream pstream = new ProgressStream(fs);
            pstream.ProgressChanged += pstream_ProgressChanged;
            pstream.SetLength(m_Blob.Properties.Length);
            m_Blob.ServiceClient.ParallelOperationThreadCount = 10;
            asyncresult = m_Blob.BeginDownloadToStream(pstream, BlobTransferCompletedCallback, new BlobTransferAsyncState(m_Blob, pstream));
        }

privatevoid pstream_ProgressChanged(object sender, ProgressChangedEventArgs e)
        {
            BlobTransferProgressChangedEventArgs eArgs = null;
int progress = (int)((double)e.BytesRead / e.TotalLength * 100);

// raise the progress changed event on the asyncop thread
            eArgs = new BlobTransferProgressChangedEventArgs(e.BytesRead, e.TotalLength, progress, CalculateSpeed(e.BytesRead), null);
            asyncOp.Post(delegate(object e2) { OnTaskProgressChanged((BlobTransferProgressChangedEventArgs)e2); }, eArgs);
        }

privatevoid BlobTransferCompletedCallback(IAsyncResult result)
        {
            BlobTransferAsyncState state = (BlobTransferAsyncState)result.AsyncState;
            ICloudBlob blob = state.Blob;
            ProgressStream stream = (ProgressStream)state.Stream;

try
            {
                stream.Close();

// End the operation.
if (TransferType == TransferTypeEnum.Download)
                    blob.EndDownloadToStream(result);
elseif (TransferType == TransferTypeEnum.Upload)
                    blob.EndUploadFromStream(result);

// Operation completed normally, raise the completed event
                AsyncCompletedEventArgs completedArgs = new AsyncCompletedEventArgs(null, false, null);
                asyncOp.PostOperationCompleted(delegate(object e) { OnTaskCompleted((AsyncCompletedEventArgs)e); }, completedArgs);
            }
catch (StorageException ex)
            {
if (!state.Cancelled)
                {
throw (ex);
                }

// Operation was cancelled, raise the event with the cancelled flag = true
                AsyncCompletedEventArgs completedArgs = new AsyncCompletedEventArgs(null, true, null);
                asyncOp.PostOperationCompleted(delegate(object e) { OnTaskCompleted((AsyncCompletedEventArgs)e); }, completedArgs);
            }
        }

// Cancel the async download
publicvoid CancelAsync()
        {
            ((BlobTransferAsyncState)asyncresult.AsyncState).Cancelled = true;
            asyncresult.Cancel();
        }

// Helper function to only raise the event if the client has subscribed to it.
protectedvirtualvoid OnTaskCompleted(AsyncCompletedEventArgs e)
        {
if (TransferCompleted != null)
                TransferCompleted(this, e);
        }

// Helper function to only raise the event if the client has subscribed to it.
protectedvirtualvoid OnTaskProgressChanged(BlobTransferProgressChangedEventArgs e)
        {
if (TransferProgressChanged != null)
                TransferProgressChanged(this, e);
        }

// Keep the last 200 progress change notifications and use them to calculate the average speed over that duration. 
privatedouble CalculateSpeed(long BytesSent)
        {
double speed = 0;

if (timeQueue.Count >= 200)
            {
                timeQueue.Dequeue();
                bytesQueue.Dequeue();
            }

            timeQueue.Enqueue(System.DateTime.Now.Ticks);
            bytesQueue.Enqueue(BytesSent);

if (timeQueue.Count > 2)
            {
                updateTime = System.DateTime.Now;
                speed = (bytesQueue.Max() - bytesQueue.Min()) / TimeSpan.FromTicks(timeQueue.Max() - timeQueue.Min()).TotalSeconds;
            }

return speed;
        }

// A modified version of the ProgressStream from http://blogs.msdn.com/b/paolos/archive/2010/05/25/large-message-transfer-with-wcf-adapters-part-1.aspx
// This class allows progress changed events to be raised from the blob upload/download.
privateclass ProgressStream : Stream
        {
#region Private Fields
private Stream stream;
privatelong bytesTransferred;
privatelong totalLength;
#endregion

#region Public Handler
publicevent EventHandler<ProgressChangedEventArgs> ProgressChanged;
#endregion

#region Public Constructor
public ProgressStream(Stream file)
            {
this.stream = file;
this.totalLength = file.Length;
this.bytesTransferred = 0;
            }
#endregion

#region Public Properties
publicoverridebool CanRead
            {
                get
                {
returnthis.stream.CanRead;
                }
            }

publicoverridebool CanSeek
            {
                get
                {
returnthis.stream.CanSeek;
                }
            }

publicoverridebool CanWrite
            {
                get
                {
returnthis.stream.CanWrite;
                }
            }

publicoverridevoid Flush()
            {
this.stream.Flush();
            }

publicoverridevoid Close()
            {
this.stream.Close();
            }

publicoverridelong Length
            {
                get
                {
returnthis.stream.Length;
                }
            }

publicoverridelong Position
            {
                get
                {
returnthis.stream.Position;
                }
                set
                {
this.stream.Position = value;
                }
            }
#endregion

#region Public Methods
publicoverrideint Read(byte[] buffer, int offset, int count)
            {
int result = stream.Read(buffer, offset, count);
                bytesTransferred += result;
if (ProgressChanged != null)
                {
try
                    {
                        OnProgressChanged(new ProgressChangedEventArgs(bytesTransferred, totalLength));
//ProgressChanged(this, new ProgressChangedEventArgs(bytesTransferred, totalLength));
                    }
catch (Exception)
                    {
                        ProgressChanged = null;
                    }
                }
return result;
            }

protectedvirtualvoid OnProgressChanged(ProgressChangedEventArgs e)
            {
if (ProgressChanged != null)
                    ProgressChanged(this, e);
            }

publicoverridelong Seek(long offset, SeekOrigin origin)
            {
returnthis.stream.Seek(offset, origin);
            }

publicoverridevoid SetLength(longvalue)
            {
                totalLength = value;
//this.stream.SetLength(value);
            }

publicoverridevoid Write(byte[] buffer, int offset, int count)
            {
this.stream.Write(buffer, offset, count);
                bytesTransferred += count;
                {
try
                    {
                        OnProgressChanged(new ProgressChangedEventArgs(bytesTransferred, totalLength));
//ProgressChanged(this, new ProgressChangedEventArgs(bytesTransferred, totalLength));
                    }
catch (Exception)
                    {
                        ProgressChanged = null;
                    }
                }
            }

protectedoverridevoid Dispose(bool disposing)
            {
                stream.Dispose();
base.Dispose(disposing);
            }

#endregion
        }

privateclass BlobTransferAsyncState
        {
public ICloudBlob Blob;
public Stream Stream;
public DateTime Started;
publicbool Cancelled;

public BlobTransferAsyncState(ICloudBlob blob, Stream stream)
                : this(blob, stream, DateTime.Now)
            { }

public BlobTransferAsyncState(ICloudBlob blob, Stream stream, DateTime started)
            {
                Blob = blob;
                Stream = stream;
                Started = started;
                Cancelled = false;
            }
        }

privateclass ProgressChangedEventArgs : EventArgs
        {
#region Private Fields
privatelong bytesRead;
privatelong totalLength;
#endregion

#region Public Constructor
public ProgressChangedEventArgs(long bytesRead, long totalLength)
            {
this.bytesRead = bytesRead;
this.totalLength = totalLength;
            }
#endregion

#region Public properties

publiclong BytesRead
            {
                get
                {
returnthis.bytesRead;
                }
                set
                {
this.bytesRead = value;
                }
            }

publiclong TotalLength
            {
                get
                {
returnthis.totalLength;
                }
                set
                {
this.totalLength = value;
                }
            }
#endregion
        }

publicenum TransferTypeEnum
        {
            Download,
            Upload
        }

publicclass BlobTransferProgressChangedEventArgs : System.ComponentModel.ProgressChangedEventArgs
        {
privatelong m_BytesSent = 0;
privatelong m_TotalBytesToSend = 0;
privatedouble m_Speed = 0;

publiclong BytesSent
            {
                get { return m_BytesSent; }
            }

publiclong TotalBytesToSend
            {
                get { return m_TotalBytesToSend; }
            }

publicdouble Speed
            {
                get { return m_Speed; }
            }

public TimeSpan TimeRemaining
            {
                get
                {
                    TimeSpan time = new TimeSpan(0, 0, (int)((TotalBytesToSend - m_BytesSent) / (m_Speed == 0 ? 1 : m_Speed)));
return time;
                }
            }

public BlobTransferProgressChangedEventArgs(long BytesSent, long TotalBytesToSend, int progressPercentage, double Speed, object userState)
                : base(progressPercentage, userState)
            {
                m_BytesSent = BytesSent;
                m_TotalBytesToSend = TotalBytesToSend;
                m_Speed = Speed;
            }
        }
    }
}

BlobTransfer transfer;

privatevoid button1_Click(object sender, EventArgs e)
{
    CloudStorageAccount account = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials("accountname", "accountkey"), false);
    CloudBlobClient client = account.CreateCloudBlobClient();
    CloudBlobContainer container = client.GetContainerReference("container");
    CloudBlockBlob blob = container.GetBlockBlobReference("file");

    transfer = new BlobTransfer();
    transfer.TransferProgressChanged += transfer_TransferProgressChanged;
    transfer.TransferCompleted += transfer_TransferCompleted;
    transfer.DownloadBlobAsync(blob, @"C:\temp\file");
}

privatevoid button2_Click(object sender, EventArgs e)
{
    transfer.CancelAsync();
}

void transfer_TransferCompleted(object sender, AsyncCompletedEventArgs e)
{
    System.Diagnostics.Debug.WriteLine("Completed. Cancelled = " + e.Cancelled);
}

void transfer_TransferProgressChanged(object sender, BlobTransfer.BlobTransferProgressChangedEventArgs e)
{
    System.Diagnostics.Debug.WriteLine("Changed - " + e.BytesSent + " / " + e.TotalBytesToSend + " = " + e.ProgressPercentage + "% " + e.Speed);
}

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.StorageClient;

namespace ConsoleApplication1
 {
class Program
     {
conststring ACCOUNTNAME = "ENTER ACCOUNT NAME";
conststring ACCOUNTKEY = "ENTER ACCOUNT KEY";
conststring LOCALFILE = @"ENTER LOCAL FILE";
conststring CONTAINER = "temp";

privatestatic CloudStorageAccount AccountFileTransfer;
privatestatic CloudBlobClient BlobClientFileTransfer;
privatestatic CloudBlobContainer ContainerFileTransfer;

privatestaticbool Transferring;

staticvoid Main(string[] args)
         {
             System.Net.ServicePointManager.DefaultConnectionLimit = 35;

             AccountFileTransfer = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + ACCOUNTNAME + ";AccountKey=" + ACCOUNTKEY);
if (AccountFileTransfer != null)
             {
                 BlobClientFileTransfer = AccountFileTransfer.CreateCloudBlobClient();
                 ContainerFileTransfer = BlobClientFileTransfer.GetContainerReference(CONTAINER);
                 ContainerFileTransfer.CreateIfNotExist();
             }

// Upload the file
             CloudBlob blobUpload = ContainerFileTransfer.GetBlobReference(CONTAINER + "/" + System.IO.Path.GetFileName(LOCALFILE));
             BlobTransfer transferUpload = new BlobTransfer();
             transferUpload.TransferProgressChanged += new EventHandler<BlobTransfer.BlobTransferProgressChangedEventArgs>(transfer_TransferProgressChanged);
             transferUpload.TransferCompleted += new System.ComponentModel.AsyncCompletedEventHandler(transfer_TransferCompleted);
             transferUpload.UploadBlobAsync(blobUpload, LOCALFILE);

             Transferring = true;
while (Transferring)
             {
                 Console.ReadLine();
             }

// Download the file
             CloudBlob blobDownload = ContainerFileTransfer.GetBlobReference(CONTAINER + "/" + System.IO.Path.GetFileName(LOCALFILE));
             BlobTransfer transferDownload = new BlobTransfer();
             transferDownload.TransferProgressChanged += new EventHandler<BlobTransfer.BlobTransferProgressChangedEventArgs>(transfer_TransferProgressChanged);
             transferDownload.TransferCompleted += new System.ComponentModel.AsyncCompletedEventHandler(transfer_TransferCompleted);
             transferDownload.DownloadBlobAsync(blobDownload, LOCALFILE + ".copy");

             Transferring = true;
while (Transferring)
             {
                 Console.ReadLine();
             }
         }

staticvoid transfer_TransferCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
         {
             Transferring = false;
             Console.WriteLine("Transfer completed. Press any key to continue.");
         }

staticvoid transfer_TransferProgressChanged(object sender, BlobTransfer.BlobTransferProgressChangedEventArgs e)
         {
             Console.WriteLine("Transfer progress percentage = " + e.ProgressPercentage + " - " + (e.Speed / 1024).ToString("N2") + "KB/s");
         }
     }
 }