1
\$\begingroup\$

This is one of my first "real" application using C#, so I'm probably doing a lot of things wrong.

I made a simple news crawler, which uses Google to search for news articles of multiple companies. You can specify a From and To date, and it will get most articles in that time range.

It uses Windows Forms, so I'm only going to post the actual code, not the Windows Forms Designer internal code. I hope that's ok.

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Net;
using Newtonsoft.Json;
using System.Threading;

namespace NewsCrawler
{
    public partial class Form1 : Form
    {
        private Dictionary<string, string> Webpages = new Dictionary<string, string>();
        public Form1()
        {
            InitializeComponent();
        }

        private void SearchButton_Click(object sender, EventArgs e)
        {
            UseWaitCursor = true;
            Webpages.Clear();
            PageResults.Items.Clear();
            AmountLabel.Text = "0";

            Task.Run(() =>
            {
                Search("google").ToList().ForEach(x => Webpages[x.Key] = x.Value);
                Search("microsoft").ToList().ForEach(x => Webpages[x.Key] = x.Value);
                Search("alphabet").ToList().ForEach(x => Webpages[x.Key] = x.Value);
                Search("nvidia").ToList().ForEach(x => Webpages[x.Key] = x.Value);
                Search("intel").ToList().ForEach(x => Webpages[x.Key] = x.Value);

                AmountLabel.Invoke((MethodInvoker)delegate
                {
                    AmountLabel.Text = Webpages.Count.ToString();
                });
                foreach (var page in Webpages)
                    PageResults.Invoke((MethodInvoker)delegate
                    {
                        PageResults.Items.Add(page.Key);
                    });
                Invoke((MethodInvoker)delegate
                {
                    UseWaitCursor = false;
                });
            });
        }

        private Dictionary<string, string> Search(string query)
        {
            var pages = new Dictionary<string, string>();
            for (int i = 0; i <= (ToDate.Value - FromDate.Value).TotalDays / 3; ++i)
            {
                var parameters = new Dictionary<string, string>
                {
                    ["key"] = "(application key removed)",
                    ["cx"] = "(search engine id removed)",
                    ["googlehost"] = "google.com",
                    ["num"] = "10",
                    ["start"] = (10 * i + 1).ToString(),
                    ["sort"] = "date:r:" + FromDate.Value.ToString("yyyyMMdd") + ":" + ToDate.Value.ToString("yyyMMdd"),
                    ["q"] = query
                };

                var url = "https://www.googleapis.com/customsearch/v1?";
                foreach (var param in parameters)
                    url += param.Key + "=" + param.Value + "&";
                url = url.Remove(url.Length - 1);

                var webClient = new WebClient();
                dynamic jsonObject = JsonConvert.DeserializeObject(webClient.DownloadString(url));

                foreach (var item in jsonObject.items)
                {
                    if (!pages.ContainsKey((string)item.title))
                        pages.Add((string)item.title, (string)item.link);
                }
            }
            return pages;
        }

        private void PageResults_SelectedIndexChanged(object sender, EventArgs e)
        {
            System.Diagnostics.Process.Start(Webpages[PageResults.Items[PageResults.SelectedIndex].ToString()]);
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            FromDate.Value = DateTime.Today.AddDays(-1);
            ToDate.Value = DateTime.Today;
            ToDate.MaxDate = DateTime.Today;
        }

        private void ToDate_ValueChanged(object sender, EventArgs e)
        {
            FromDate.MaxDate = ToDate.Value;
        }
    }
}
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.