• R/O
  • SSH
  • HTTPS

openpishcrawler: Commit


Commit MetaInfo

Revision3 (tree)
Time2019-10-10 08:07:21
Authorsebastiandotnet

Log Message

- improve sleeping page check
- fix dublettes check

Change Summary

Incremental Difference

--- OpenPishCrawler/Data/PishingPageList.cs (revision 2)
+++ OpenPishCrawler/Data/PishingPageList.cs (revision 3)
@@ -24,7 +24,7 @@
2424
2525 public new void Add(PishingPage page)
2626 {
27- if (!Items.Any(e => e.Url.Equals(page.Url)))
27+ if (!Items.Any(e => e.Url.ToWebHostAppSubdomain().Equals(page.Url.ToWebHostAppSubdomain())))
2828 base.Add(page);
2929 }
3030
--- OpenPishCrawler/Extensions/StringExtensions.cs (revision 2)
+++ OpenPishCrawler/Extensions/StringExtensions.cs (revision 3)
@@ -41,5 +41,21 @@
4141
4242 return result;
4343 }
44+
45+ public static string ToWebHostAppSubdomain(this string value)
46+ {
47+ var result = value.Trim();
48+ foreach (var item in Prefix)
49+ {
50+ if (result.StartsWith(item, StringComparison.InvariantCultureIgnoreCase))
51+ result = result.Substring(item.Length);
52+ }
53+
54+ var array = result.Split(new string[] { "." }, StringSplitOptions.RemoveEmptyEntries);
55+ if (array.Length >= 3)
56+ result = array[0];
57+
58+ return result;
59+ }
4460 }
4561 }
--- OpenPishCrawler/Service/OpenPishReader.cs (revision 2)
+++ OpenPishCrawler/Service/OpenPishReader.cs (revision 3)
@@ -61,6 +61,8 @@
6161
6262 private string ReadOpenPishPage()
6363 {
64+ return File.ReadAllText(@"C:\Users\FACE2FACE\Documents\Visual Studio 2015\Projects\OpenPishCrawler\DebugTrace\OpenPishExample.html");
65+
6466 var result = default(string);
6567 using (var webClient = new WebClient())
6668 {
Show on old repository browser