1 namespace CrawlerCPU
 2 {
 3     /* 根据规格，一个服务爬取3个导航页（由 中央处理器 统一管理）
 4      * 第一个服务：日韩时装，港台时装
 5      * 第二个服务：，欧美时装，明星穿衣，显瘦搭配
 6      * 第三个服务：少女搭配，职场搭配，裙装搭配
 7      */
 8     public class Program
 9     {
10         static Dictionary<string, string> dicMapping = new Dictionary<string, string>();
11 
12         static void Main(string[] args)
13         {
14             //初始Url的分发
15             foreach (var key in ConfigurationManager.AppSettings)
16             {
17                 var factory = new ChannelFactory<ICrawlerService>(new NetTcpBinding(), new EndpointAddress(key.ToString()))
18                                  .CreateChannel();
19 
20                 var urls = ConfigurationManager.AppSettings[key.ToString()]
21                                                .Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
22                                                .ToList();
23 
24                 factory.AddRange(urls);
25 
26                 //将网址和“WCF分发地址“建立Mapping映射
27                 foreach (var item in urls)
28                     dicMapping.Add(item, key.ToString());
29             }
30 
31             Console.WriteLine("爬虫 中央处理器开启，正在监视TODO列表！");
32 
33             //开启定时监视MongoDB
34             Timer timer = new Timer();
35 
36             timer.Interval = 1000 * 10; //10s轮询一次
37             timer.Elapsed += new ElapsedEventHandler(timer_Elapsed);
38             timer.Start();
39 
40             Console.Read();
41         }
42 
43         static void timer_Elapsed(object sender, ElapsedEventArgs e)
44         {
45             //获取mongodb里面的数据
46             MongodbHelper<Message> mongodb = new MongodbHelper<Message>("todo");
47 
48             //根据url的类型分发到相应的服务器中去处理
49             var urls = mongodb.List(100);
50 
51             if (urls == null || urls.Count == 0)
52                 return;
53 
54             foreach (var item in dicMapping.Keys)
55             {
56                 foreach (var url in urls)
57                 {
58                     //寻找正确的 wcf 分发地址
59                     if (url.Url.StartsWith(item))
60                     {
61                         var factory = new ChannelFactory<ICrawlerService>(new NetTcpBinding(),
62                                       new EndpointAddress(dicMapping[item]))
63                                       .CreateChannel();
64 
65                         //向正确的地方分发地址
66                         factory.Add(url.Url);
67 
68                         break;
69                     }
70                 }
71             }
72 
73             //删除mongodb中的TODO表中指定数据
74             mongodb.Remove(urls);
75         }
76     }
77 }

 1         /// <summary>
 2         /// 开始执行任务
 3         /// </summary>
 4         public static void Start()
 5         {
 6             while (true)
 7             {
 8                 //监视工作线程，如果某个线程已经跑完数据，则重新分配任务给该线程
 9                 for (int j = 0; j < 10; j++)
10                 {
11                     if (tasks[j] == null || tasks[j].IsCompleted || tasks[j].IsCanceled || tasks[j].IsFaulted)
12                     {
13                         //如果队列还有数据
14                         if (todoQueue.Count > 0)
15                         {
16                             string currentUrl = string.Empty;
17 
18                             todoQueue.TryDequeue(out currentUrl);
19 
20                             Console.WriteLine("当前队列的个数为:{0}", todoQueue.Count);
21 
22                             tasks[j] = Task.Factory.StartNew((obj) =>
23                             {
24                                 DownLoad(obj.ToString());
25 
26                             }, currentUrl);
27                         }
28                     }
29                 }
30             }
31         }