1
\$\begingroup\$

The returned class as result:

 public class cs_HttpFetchResults
    {
        public bool blResultSuccess = false;
        public string srFetchBody = "";
        public string srFetchingFinalURL = "";
        public bool bl404 = false;
    }

The HttpWebRequest version:

    public static cs_HttpFetchResults func_fetch_Page(string srUrl, int irTimeOut = 60,
 string srRequestUserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0",
 string srProxy = null, int irCustomEncoding = 0, bool blAutoDecode = true, bool blKeepAlive = true,
  string srIPandHost = null)
    {
        cs_HttpFetchResults mycs_HttpFetchResults = new cs_HttpFetchResults();

        mycs_HttpFetchResults.srFetchingFinalURL = srUrl;

        HttpWebRequest request = null;
        WebResponse response = null;

        try
        {
            request = (HttpWebRequest)WebRequest.Create(srUrl);
            request.CookieContainer = new System.Net.CookieContainer();

            if (srProxy != null)
            {
                string srProxyHost = srProxy.Split(':')[0];
                int irProxyPort = Int32.Parse(srProxy.Split(':')[1]);
                System.Net.WebProxy my_awesomeproxy = new WebProxy(srProxyHost, irProxyPort);
                my_awesomeproxy.Credentials = new NetworkCredential();
                request.Proxy = my_awesomeproxy;
            }
            else
            {
                request.Proxy = null;
            }

            request.ContinueTimeout = irTimeOut * 1000;
            request.ReadWriteTimeout = irTimeOut * 1000;
            request.Timeout = irTimeOut * 1000;
            request.UserAgent = srRequestUserAgent;
            request.KeepAlive = blKeepAlive;
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";

            WebHeaderCollection myWebHeaderCollection = request.Headers;
            myWebHeaderCollection.Add("Accept-Language", "en-gb,en;q=0.5");
            myWebHeaderCollection.Add("Accept-Encoding", "gzip, deflate");

            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;

            using (response = request.GetResponse())
            {
                using (Stream strumien = response.GetResponseStream())
                {
                    Encoding myEncoding = Encoding.UTF8;
                    string srContentType = "";

                    if (response.ContentType != null)
                    {
                        srContentType = response.ContentType;
                        if (srContentType.Contains(";"))
                        {
                            srContentType = srContentType.Split(';')[1];
                        }
                        srContentType = srContentType.Replace("charset=", "");
                        srContentType = func_Process_Html_Input(srContentType);
                    }

                    try
                    {
                        myEncoding = Encoding.GetEncoding(srContentType);
                    }
                    catch
                    {
                        myEncoding = irCustomEncoding == 0 ? Encoding.UTF8
                                                         : Encoding.GetEncoding(irCustomEncoding);
                    }

                    using (StreamReader sr = new StreamReader(strumien, myEncoding))
                    {
                        mycs_HttpFetchResults.srFetchBody = sr.ReadToEnd();
                        if (blAutoDecode == true)
                        {
                            mycs_HttpFetchResults.srFetchBody = HttpUtility.HtmlDecode(mycs_HttpFetchResults.srFetchBody);
                        }
                        mycs_HttpFetchResults.srFetchingFinalURL = Return_Absolute_Url(response.ResponseUri.AbsoluteUri.ToString(), response.ResponseUri.AbsoluteUri.ToString());
                        mycs_HttpFetchResults.blResultSuccess = true;
                    }
                }
            }

            if (request != null)
                request.Abort();
            request = null;
        }
        catch (Exception E)
        {
            if (E.Message.ToString().Contains("(404)"))
                mycs_HttpFetchResults.bl404 = true;

            csLogger.logCrawlingErrors("crawling failed url: " + srUrl, E);

        }
        finally
        {
            if (request != null)
                request.Abort();
            request = null;

            if (response != null)
                response.Close();
            response = null;
        }

        return mycs_HttpFetchResults;
    }

The HttpClientHandler:

    async public static Task<cs_HttpFetchResults> func_fetch_Page_New(string srUrl, int irTimeOut = 60,
  string srRequestUserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0",
  string srProxy = null, int irCustomEncoding = 0, bool blAutoDecode = true, bool blKeepAlive = true,
  string srIPandHost = null)
    {
        cs_HttpFetchResults mycs_HttpFetchResults = new cs_HttpFetchResults();

        mycs_HttpFetchResults.srFetchingFinalURL = srUrl;

        try
        {
            using (HttpClientHandler myClientHandler = new HttpClientHandler())
            {
                myClientHandler.AllowAutoRedirect = true;
                myClientHandler.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                myClientHandler.UseDefaultCredentials = true;

                if (srProxy != null)
                {
                    string srProxyHost = srProxy.Split(':')[0];
                    int irProxyPort = Int32.Parse(srProxy.Split(':')[1]);
                    System.Net.WebProxy my_awesomeproxy = new WebProxy(srProxyHost, irProxyPort);
                    my_awesomeproxy.Credentials = new NetworkCredential();
                    myClientHandler.Proxy = my_awesomeproxy;
                    myClientHandler.UseProxy = true;
                }
                else
                {
                    myClientHandler.Proxy = null;
                    myClientHandler.UseProxy = false;
                }

                using (var httpClient = new HttpClient(myClientHandler))
                {
                    httpClient.DefaultRequestHeaders.Add("Accept-Language", "en-gb,en;q=0.5");
                    httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
                    httpClient.Timeout = new TimeSpan(0, 0, irTimeOut);
                    httpClient.DefaultRequestHeaders.Add("User-Agent", srRequestUserAgent);
                    httpClient.DefaultRequestHeaders.Add("User-Agent", srRequestUserAgent);

                    if (blKeepAlive == true)
                    {
                        httpClient.DefaultRequestHeaders.Connection.Clear();
                        httpClient.DefaultRequestHeaders.ConnectionClose = false;
                        httpClient.DefaultRequestHeaders.Connection.Add("Keep-Alive");
                    }
                    else
                    {
                        httpClient.DefaultRequestHeaders.Connection.Clear();
                        httpClient.DefaultRequestHeaders.ConnectionClose = true;
                    }

                    using (var vrResponse = await httpClient.GetAsync(srUrl))
                    {
                        if (vrResponse.IsSuccessStatusCode == true)
                        {
                            var contenttype = vrResponse?.Content?.Headers?.First(h => h.Key.Equals("Content-Type"));
                            string srContentType = contenttype?.Value?.First();

                            Encoding myEncoding = Encoding.UTF8;

                            if (srContentType != null)
                            {
                                if (srContentType.Contains(";"))
                                {
                                    srContentType = srContentType.Split(';')[1];
                                }
                                srContentType = srContentType.Replace("charset=", "");
                                srContentType = func_Process_Html_Input(srContentType);
                            }

                            try
                            {
                                myEncoding = Encoding.GetEncoding(srContentType);
                            }
                            catch
                            {
                                myEncoding = irCustomEncoding == 0 ? Encoding.UTF8
                                                                 : Encoding.GetEncoding(irCustomEncoding);
                            }

                            var bytes = await vrResponse.Content.ReadAsByteArrayAsync();
                            mycs_HttpFetchResults.srFetchBody = myEncoding.GetString(bytes);
                            if (blAutoDecode == true)
                            {
                                mycs_HttpFetchResults.srFetchBody = HttpUtility.HtmlDecode(mycs_HttpFetchResults.srFetchBody);
                            }

                            string responseUri = vrResponse.RequestMessage.RequestUri.AbsoluteUri.ToString();

                            mycs_HttpFetchResults.srFetchingFinalURL = Return_Absolute_Url(responseUri, responseUri);
                            mycs_HttpFetchResults.blResultSuccess = true;
                        }
                        else
                        {
                            if (vrResponse.StatusCode == HttpStatusCode.NotFound)
                                mycs_HttpFetchResults.bl404 = true;
                        }
                    }
                }
            }
        }
        catch (Exception E)
        {
            if (E.Message.ToString().Contains("(404)"))
                mycs_HttpFetchResults.bl404 = true;

            csLogger.logCrawlingErrors("crawling failed url: " + srUrl, E);
        }

        return mycs_HttpFetchResults;
    }

Other helper methods:

public static string func_Process_Html_Input(string srHtmlInput)
    {
        srHtmlInput = HttpUtility.HtmlDecode(srHtmlInput);
        srHtmlInput = Regex.Replace(srHtmlInput, @"(\s)\s+", "$1").Trim();
        srHtmlInput = srHtmlInput.Replace("&#39", "'");
        return srHtmlInput;
    }

    public static string Return_Absolute_Url(string srRelativeUrl, string srCrawledUrl, List<string> lstBannedExtensions = null, bool blIgnoreBaseUriHost = false, bool blDoNotRemoveDash = false)
    {

        srRelativeUrl = HttpUtility.UrlDecode(srRelativeUrl);

        srRelativeUrl = HttpUtility.HtmlDecode(srRelativeUrl);

        lstBannedExtensions = (lstBannedExtensions == null) ? new List<String>() : lstBannedExtensions;

        string srReturnUrl = "null";

        if (srRelativeUrl.Length > 0)
            if (srRelativeUrl[0] == '.')
                srRelativeUrl = srRelativeUrl.Substring(1);

        Uri baseUri = new Uri(srCrawledUrl);

        Uri NewUrl;

        bool blUriResult = Uri.TryCreate(baseUri, srRelativeUrl, out NewUrl);

        if (blUriResult == true)
        {
            if (NewUrl.AbsoluteUri.ToString().StartsWith("http") && (NewUrl.Host == baseUri.Host || blIgnoreBaseUriHost == true))
            {
                string srLastSegment = NewUrl.Segments[NewUrl.Segments.Length - 1].ToString();
                if (lstBannedExtensions.Where(pr => srLastSegment.ToLowerInvariant().IndexOf(pr) != -1).Count<string>() == 0)
                {
                    srRelativeUrl = NewUrl.AbsoluteUri.ToString();
                    if (srRelativeUrl.IndexOf("#") != -1 && blDoNotRemoveDash == false)
                    {
                        srRelativeUrl = srRelativeUrl.Substring(0, srRelativeUrl.IndexOf("#"));
                    }

                    srRelativeUrl = HttpUtility.UrlDecode(srRelativeUrl);

                    srRelativeUrl = HttpUtility.HtmlDecode(srRelativeUrl);

                    srReturnUrl = srRelativeUrl;
                }
            }
        }

        return srReturnUrl;
    }
\$\endgroup\$
3
\$\begingroup\$

You really need to follow Microsoft's naming conventions.

What is even the point of prefixing your class names with "cs"? All you get is unreadable names, especially when you then name your variables "my(ClassName)", e.g. mycs_HttpFetchResults. Using the "my" prefix is something you should avoid as well.

Same with properties etc.: do not implement some version of an Hungarian notation.


func_fetch_Page has eight parameters. That's waaay too many. Instead, construct a class where each of those params is a property, and pass that class.


If you have an if...else, try to make your if "positive":

        if (srProxy == null)
        {
           // create srProxy
        }
        else
        {
           // whatever is necessary
        }

Split your methods into smaller ones. func_fetch_Page is 120 lines, you could for instance easily move the creation of HttpWebRequest to its own method. Think SOLID.


Adding "New" to a method name -- as in func_fetch_Page_New -- is just bad, unless this is a temporary measure and you need to check whether your new method does the same as your old one. Why not call this one FetchPageAsync?


Don't name a method "Return(Something)", e.g. Return_Absolute_Url. Name it Get(Something)".

\$\endgroup\$
  • \$\begingroup\$ ty for suggestions. it seems like my most mistakes about naming. do functions looks solid? also passing a lot of parameters have any problem performance wise? Actually i would appreciate if you rename them. So i would see how they would look when properly named :) \$\endgroup\$ – MonsterMMORPG Mar 21 '17 at 14:49

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.