One of the challenges of working with queues is how to properly and neatly process each item using multiple workers, luckily on .Net we have Semaphores. “In computer science, a semaphore is a variable or abstract data type that is used for controlling access, by multiple processes, to a common resource in a concurrent system such as a multiprogramming operating system.” https://en.wikipedia.org/wiki/Semaphore_(programming)
Below is a simple program that demonstrate the use of semaphore with a parametized number of threads and HttpRequest as Payload.
References:
https://msdn.microsoft.com/en-us/library/system.threading.semaphore(v=vs.110).aspx
private const int maxThreads = 10;
private Semaphore semaphore = new Semaphore(maxThreads, maxThreads);
private Queue queue = new Queue();
private void button1_Click(object sender, EventArgs e)
{
addTestData();
while (queue.Count >=1)
{
semaphore.WaitOne();
var item = queue.Dequeue();
ThreadPool.QueueUserWorkItem(ProcessCurrentItem, item);
}
int count = 0;
while (count < maxThreads)
{
semaphore.WaitOne();
++count;
}
}
async void ProcessCurrentItem(object item)
{
string responseData = "";
HttpClient hc = new HttpClient();
hc.DefaultRequestHeaders.Add("host", "www.mywebsite.com");
hc.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");
hc.DefaultRequestHeaders.Add("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
hc.DefaultRequestHeaders.Add("scheme", "https");
hc.DefaultRequestHeaders.Add("method", "GET");
//Get the headers associated with the request.
try
{
HttpResponseMessage response = await hc.GetAsync("https://www.mywebsite.com/" + item.ToString());
responseData = response.ToString();
//DO WHAT YOU WANT WITH YOUR DATA HERE
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
Console.WriteLine(item.ToString());
semaphore.Release();
}
private void addTestData()
{
queue.Enqueue("1.json");
queue.Enqueue("2.json");
queue.Enqueue("3.json");
queue.Enqueue("4.json");
queue.Enqueue("5.json");
queue.Enqueue("6.json");
queue.Enqueue("7.json");
queue.Enqueue("8.json");
queue.Enqueue("9.json");
queue.Enqueue("10.json");
queue.Enqueue("11.json");
queue.Enqueue("12.json");
queue.Enqueue("13.json");
queue.Enqueue("14.json");
queue.Enqueue("15.json");
queue.Enqueue("16.json");
queue.Enqueue("17.json");
queue.Enqueue("18.json");
}