Here a simple example of how I can upload very large file from my local file system to an Azure Blob Storage.
I collected few chunks of code found on the network and I create a console application which can perform this task.
I found some example where developer split large file into several blocks and perform a lot of little call manually handling the result of each call.
I tried to understand what the UploadFromStreamAsync method works with big files and how I can monitor the upload status.
Behind the scenes, this method automatically splits the file in chunk of about 4 MByte everyone and performs a lot of parallel call to the Azure Blob Storage.
Here my code, released as is…
And below a simple video where you can see what’s happen while the file is uploaded.
I apologize if there is a misprint in a sign, but I do not really want to redo the video
:)
I apologize if there is a misprint in a sign, but I do not really want to redo the video
:)
Be careful to add the following nuget packages and add to the app.config file the following configuration replacing the right Azure configuration values...
appSettings>
add key="StorageConnectionString" value="DefaultEndpointsProtocol=https;AccountName=thisishtenameoftheaccount;AccountKey=S0AaU6Xcf21mp+cEhztDSOK/HzW7Q8IqhCIAOJNe9VDsmPaS+mAMmazWLETEtteQvTc0EimI7xFNwFyplWTMw==" />
appSettings>
usingMicrosoft.Azure;
usingMicrosoft.WindowsAzure.Storage;
usingMicrosoft.WindowsAzure.Storage.Blob;
usingMicrosoft.WindowsAzure.Storage.RetryPolicies;
using System;
using System.IO;
usingSystem.Threading;
usingSystem.Threading.Tasks;
namespace BlobTest
{
class Program
{
private static object Lock = new object();
private static Int64 Size = 0;
private static DateTime StartDate;
private static CancellationTokenSource TokenSource = new CancellationTokenSource();
private static CancellationToken CancellationToken = TokenSource.Token;
static void Main(string[] args)
{
const string FILENAME = @"e:\vm\uStudio.vhdx";
const string CONTAINER = "testbigfiles";
const string BLOB = "folder1\\uStudio.vhdx";
uplodadFile(CONTAINER, BLOB, FILENAME);
Console.ReadLine();
}
private static void uplodadFile(string containerName, string blobName, string fileName)
{
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blob = container.GetBlockBlobReference(blobName);
//Setting BlobRequestOptions
BlobRequestOptions bro = new BlobRequestOptions()
{
//How many call can perform at the same time
ParallelOperationThreadCount = 64, //Max Value
//The retry policy (automatically managed)
RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(1), 64)
};
//Openig the file
using (var fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
//Creating the operation context to manage the status of the requests
var operationContext = new OperationContext();
//Adding event receiver on "Response Received" on order to display upload information
operationContext.ResponseReceived += OperationContext_ResponseReceived;
//Setting StartDate
StartDate = DateTime.Now;
//Uploading Asyncronously the file
var upload = blob.UploadFromStreamAsync(fileStream, new AccessCondition(), bro, operationContext, CancellationToken);
//When completed successfully completed...
upload.ContinueWith(task =>
{
Console.WriteLine($"Status {task.Status}.");
Console.WriteLine("Upload is over successfully.");
}, TaskContinuationOptions.OnlyOnRanToCompletion);
//When completed with error...
upload.ContinueWith(task =>
{
Console.WriteLine($"Status {task.Status}.");
if (task.Exception != null)
{
Console.WriteLine("Task could not be completed." + task.Exception.InnerException);
}
}, TaskContinuationOptions.OnlyOnFaulted);
//Wait until upload is completed or cancelled
try
{
upload.Wait(CancellationToken);
}
catch (OperationCanceledException)
{
Console.WriteLine("Task cancelled");
}
}
}
private static voidOperationContext_ResponseReceived(object sender, RequestEventArgs e)
{
lock (Lock)
{
if (e != null && e.RequestInformation != null)
{
if (e.RequestInformation.HttpStatusCode == 201)
{
Size += e.RequestInformation.EgressBytes;
//Log info only every 2 second
if (e.RequestInformation.StartTime.Second % 2 == 0)
{
//Logging some information...
//I could have committed more :)))
var sizeMB = Size / 1024 / 1024;
var elapsed = DateTime.Now - StartDate;
Console.Clear();
Console.WriteLine("Transferred (MByte): " + (sizeMB).ToString());
Console.WriteLine("Elapsed (minutes): " + (elapsed).TotalMinutes.ToString());
Console.WriteLine("Avg Transfer rate (MByte/s): " + (sizeMB) / (elapsed).TotalSeconds);
}
}
}
}
}
}
}
