1: using System;

    2: using System.Collections.Generic;

    3: using System.IO;

    4: using System.Linq;

    5: using System.Reflection;

    6: using System.Xml.Linq;

    7: using System.Text.RegularExpressions;

    8: using OpenQA.Selenium;

    9: using OpenQA.Selenium.Chrome;

   10: using OpenQA.Selenium.Interactions;

   11: using System.Threading;

   12:  

   13: namespace Ia.Statistics.Cl.Models.SultanCenter

   14: {

   15:     ////////////////////////////////////////////////////////////////////////////

   16:  

   17:     /// <summary publish="true">

   18:     /// Structure of the sultan-center.com website.

   19:     /// </summary>

   20:     /// <remarks> 

   21:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.

   22:     ///

   23:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by

   24:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.

   25:     ///

   26:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of

   27:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

   28:     /// 

   29:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.

   30:     /// 

   31:     /// Copyright notice: This notice may not be removed or altered from any source distribution.

   32:     /// </remarks> 

   33:     public class Default

   34:     {

   35:         private static int updateProductUrlListCounter = 0;

   36:  

   37:         private static XDocument xd;

   38:  

   39:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;

   40:  

   41:         //private static Dictionary<Ia.Statistics.Cl.Model.Category, string> categoryToNextUrlDictionary = new Dictionary<Ia.Statistics.Cl.Model.Category, string>();

   42:  

   43:         private static List<string> productUrlList = new List<string>();

   44:  

   45:         ////////////////////////////////////////////////////////////////////////////

   46:  

   47:         /// <summary>

   48:         /// 

   49:         /// </summary>

   50:         public Default() { }

   51:  

   52:         ////////////////////////////////////////////////////////////////////////////

   53:  

   54:         /// <summary>

   55:         ///

   56:         /// </summary>

   57:         public static void UpdateProductUrlList(Ia.Statistics.Cl.Models.Site site)

   58:         {

   59:             var list = ProductUrlListOfCategorySequentially();

   60:  

   61:             var storedList = Ia.Statistics.Cl.Models.Product.UrlListBySiteId(site.Id);

   62:  

   63:             if (productUrlList.Count > 0)

   64:             {

   65:                 foreach (var l in list)

   66:                 {

   67:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);

   68:                 }

   69:             }

   70:             else productUrlList = list;

   71:  

   72:             if (productUrlList.Count > 0)

   73:             {

   74:                 foreach (var l in storedList)

   75:                 {

   76:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);

   77:                 }

   78:             }

   79:             else productUrlList = storedList;

   80:         }

   81:  

   82:         ////////////////////////////////////////////////////////////////////////////

   83:  

   84:         /// <summary>

   85:         ///

   86:         /// </summary>

   87:         private static List<string> ProductUrlListOfCategorySequentially()

   88:         {

   89:             Ia.Statistics.Cl.Models.Category category;

   90:  

   91:             /*

   92:             if (categoryToNextUrlDictionary.Count > 0)

   93:             {

   94:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();

   95: 

   96:                 category = categoryToNextUrl.Key;

   97:                 category.Url = categoryToNextUrl.Value;

   98: 

   99:                 categoryToNextUrlDictionary.Remove(category);

  100:             }

  101:             else

  102:             {

  103:             */

  104:             category = Ia.Statistics.Cl.Models.SultanCenter.Default.CategoryList.RandomThenNext();

  105:             //}

  106:  

  107:             var list = ProductUrlListOfCategory(category);

  108:  

  109:             return list;

  110:         }

  111:  

  112:         ////////////////////////////////////////////////////////////////////////////

  113:  

  114:         /// <summary>

  115:         ///

  116:         /// </summary>

  117:         public static List<string> ProductUrlListOfCategory(Ia.Statistics.Cl.Models.Category category)

  118:         {

  119:             var counter = 0;

  120:             var exceptionCounter = 0;

  121:  

  122:             var list = new List<string>();

  123:             var listList = new List<List<string>>();

  124:  

  125:             var options = new ChromeOptions();

  126:  

  127:             var chromeDriverService = ChromeDriverService.CreateDefaultService();

  128:             chromeDriverService.HideCommandPromptWindow = true;

  129:             chromeDriverService.SuppressInitialDiagnosticInformation = true;

  130:  

  131:             options.AddArgument("--headless");

  132:             options.AddArgument("--log-level=3");

  133:             options.AddArgument("--disable-extensions");

  134:             options.AddArgument("--ignore-certificate-errors");

  135:  

  136:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))

  137:             {

  138:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;

  139:  

  140:                 chromeDriver.Navigate().GoToUrl(category.Url);

  141:  

  142:                 // I can't suppress delivery area selection with cookies

  143:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("unbxd.userId", "uid-1717900855992-" + areaId, "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  144:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("unbxd.visit", "first_time", "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  145:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("tsc_session", "eyJpdiI6ImJyaU0rRjdpT0lDU2FVazBwZlowblE9PSIsInZhbHVlIjoiODR4OUUvOVFLL3ZUNFJncTZ0cWN4WTl6YWxtb2RSMStwQWthektlR3NKQm0zbE9XZ0JUMEU3NzB5c1hrdENjcjM2SUxYcFZxaEg3OTNKQXd0MHZqRkF0N2x1Z0hRZDBORXdiUTVWbW9tTXBSR3Q4VkVKNzNwQkNjSUg2VCtlaW0iLCJtYWMiOiI2MTA1MzNmMjcwMWMzZjQ4ZmRlNjU3MjJkMjg4NjM3MjZjNzJlMGIxMjQzMDgxYTAxMmQ0ZGQzOWM5NThkZGZhIiwidGFnIjoiIn0%3D", "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  146:  

  147:                 Thread.Sleep(2000); // important

  148:  

  149:                 try

  150:                 {

  151:                     // <input type="text" placeholder="Search in delivery area" class="delivery-input">

  152:                     var selectDeliveryAreaDropDownList = chromeDriver.FindElement(By.XPath("//input[@class='delivery-input']"), 10);

  153:  

  154:                     Actions actions = new Actions(chromeDriver);

  155:                     actions.MoveToElement(selectDeliveryAreaDropDownList).Click().Build().Perform();

  156:                     Thread.Sleep(2000);

  157:  

  158:                     // <li class="autocomplete-result"> Abdally</li >

  159:                     var selectDeliveryAreaDropDownListFirstSelection = chromeDriver.FindElement(By.XPath("//li[@class='autocomplete-result']"), 10);

  160:  

  161:                     actions.MoveToElement(selectDeliveryAreaDropDownListFirstSelection).Click().Build().Perform();

  162:                     Thread.Sleep(2000);

  163:  

  164:                     // <button type="submit" value="save store" id="submit_process" name="save_store" class="action submit primary"><span>Continue shopping</span></button>

  165:                     var continueShoppingButton = chromeDriver.FindElement(By.XPath("//button[@id='submit_process']"), 10);

  166:  

  167:                     actions.MoveToElement(continueShoppingButton).Click().Build().Perform();

  168:                     Thread.Sleep(2000);

  169:  

  170:                     var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);

  171:  

  172:                     do

  173:                     {

  174:                         javaScriptExecutor.ExecuteScript("window.scrollTo(0, document.body.scrollHeight)");

  175:  

  176:                         var content = chromeDriver.PageSource;

  177:  

  178:                         list = ProductUrlListFromCategoryContent(category, content);

  179:  

  180:                         listList.Add(list);

  181:  

  182:                         try

  183:                         {

  184:                             // enabled: <div class="pagination-box"><ul data-v-82963a40="" class="pagination"><!----> <li data-v-82963a40="" class="disabled"><a data-v-82963a40="" tabindex="-1">&lt;</a></li> <li data-v-82963a40="" class="page-item active"><a data-v-82963a40="" tabindex="0">1</a></li><li data-v-82963a40="" class="page-item"><a data-v-82963a40="" tabindex="0">2</a></li><li data-v-82963a40="" class="page-item"><a data-v-82963a40="" tabindex="0">3</a></li><li data-v-82963a40="" class="page-item disabled"><a data-v-82963a40="" tabindex="0" class="">…</a></li> <li data-v-82963a40="" class=""><a data-v-82963a40="" tabindex="0">&gt;</a></li> <!----></ul></div>

  185:                             // disabled: <div...> ... <li data-v-82963a40="" class="disabled"><a data-v-82963a40="" tabindex="-1">&gt;</a>

  186:                             var nextPageButton = chromeDriver.FindElement(By.XPath("//*[@class='pagination-box']/ul/li[not(@class='disabled')]/a[text()='>']"), 10);

  187:  

  188:                             actions.MoveToElement(nextPageButton).Click().Build().Perform();

  189:                             Thread.Sleep(2000);

  190:                         }

  191:                         catch (Exception e)

  192:                         {

  193:                             if (e.Message.Contains("no such element: Unable to locate element:")) break;

  194:  

  195:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);

  196:                         }

  197:                     }

  198:                     while (counter++ < 100);

  199:                 }

  200:                 catch (Exception e)

  201:                 {

  202:                     if (e.Message.Contains("no such element: Unable to locate element:")) { }

  203:  

  204:                     if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);

  205:                 }

  206:             }

  207:  

  208:             return listList.SelectMany(u => u).ToList();

  209:         }

  210:  

  211:         ////////////////////////////////////////////////////////////////////////////

  212:  

  213:         /// <summary>

  214:         ///

  215:         /// </summary>

  216:         public static List<string> ProductUrlListFromCategoryContent(Ia.Statistics.Cl.Models.Category category, string content)

  217:         {

  218:             string url;

  219:  

  220:             var list = new List<string>();

  221:  

  222:             /*

  223:             // below: check if content has a Next link

  224:             var nextMatch = Regex.Match(content, @"em-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);

  225: 

  226:             if (nextMatch.Success)

  227:             {

  228:                 var nextUrl = nextMatch.Groups[1].Value;

  229: 

  230:                 categoryToNextUrlDictionary[category] = nextUrl;

  231:             }

  232:             */

  233:  

  234:             var matchCollection = Regex.Matches(content, @"<div class=""proddeal_img""><a href=""(.+?)"">", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);

  235:  

  236:             foreach (Match match in matchCollection)

  237:             {

  238:                 url = match.Groups[1].Value;

  239:  

  240:                 list.Add(url);

  241:             }

  242:  

  243:             return list;

  244:         }

  245:  

  246:         ////////////////////////////////////////////////////////////////////////////

  247:         ////////////////////////////////////////////////////////////////////////////

  248:  

  249:         /// <summary>

  250:         ///

  251:         /// </summary>

  252:         public static string ReadUpdateProductPriceStockByProductUrlSequentially(Ia.Statistics.Cl.Models.Site site)

  253:         {

  254:             var s = string.Empty;

  255:             var productUrl = string.Empty;

  256:  

  257:             if (productUrlList.Count == 0 || updateProductUrlListCounter++ % Ia.Statistics.Cl.Models.Default.AverageNumberOfProductsPerCategory == 0) Ia.Statistics.Cl.Models.SultanCenter.Default.UpdateProductUrlList(site);

  258:  

  259:             if (productUrlList.Count > 0)

  260:             {

  261:                 productUrl = productUrlList.RandomThenNext();

  262:  

  263:                 ReadProductPriceStockByProductUrl(site, productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock);

  264:  

  265:                 if (!string.IsNullOrEmpty(product.Sku))

  266:                 {

  267:                     Ia.Statistics.Cl.Models.Product.Create(product);

  268:                     Ia.Statistics.Cl.Models.ProductPriceSpot.Create(product, price);

  269:                     Ia.Statistics.Cl.Models.ProductStockSpot.Create(product, stock);

  270:  

  271:                     s = "Product: " + product.Name + "; price: " + price + "; stock: " + stock + ". ";

  272:                 }

  273:                 else s = "Product: " + product.Name + ": SKU is null. ";

  274:             }

  275:             else s = "Product list is empty. ";

  276:  

  277:             s = site.Name + ": " + productUrl + ": " + s;

  278:  

  279:             return s;

  280:         }

  281:  

  282:         ////////////////////////////////////////////////////////////////////////////

  283:  

  284:         /// <summary>

  285:         ///

  286:         /// </summary>

  287:         public static void ReadProductPriceStockByProductUrl(Ia.Statistics.Cl.Models.Site site, string productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock)

  288:         {

  289:             var counter = 0;

  290:             var exceptionCounter = 0;

  291:  

  292:             var options = new ChromeOptions();

  293:  

  294:             var chromeDriverService = ChromeDriverService.CreateDefaultService();

  295:             chromeDriverService.HideCommandPromptWindow = true;

  296:             chromeDriverService.SuppressInitialDiagnosticInformation = true;

  297:  

  298:             options.AddArgument("--headless");

  299:             options.AddArgument("--log-level=3");

  300:             options.AddArgument("--disable-extensions");

  301:             options.AddArgument("--ignore-certificate-errors");

  302:  

  303:             price = stock = 0;

  304:  

  305:             product = new Ia.Statistics.Cl.Models.Product();

  306:  

  307:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))

  308:             {

  309:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;

  310:  

  311:                 chromeDriver.Navigate().GoToUrl(productUrl);

  312:  

  313:                 Thread.Sleep(2000); // important

  314:  

  315:                 try

  316:                 {

  317:                     // <input type="text" placeholder="Search in delivery area" class="delivery-input">

  318:                     var selectDeliveryAreaDropDownList = chromeDriver.FindElement(By.XPath("//input[@class='delivery-input']"), 10);

  319:  

  320:                     Actions actions = new Actions(chromeDriver);

  321:                     actions.MoveToElement(selectDeliveryAreaDropDownList).Click().Build().Perform();

  322:                     Thread.Sleep(2000);

  323:  

  324:                     // <li class="autocomplete-result"> Abdally</li >

  325:                     var selectDeliveryAreaDropDownListFirstSelection = chromeDriver.FindElement(By.XPath("//li[@class='autocomplete-result']"), 10);

  326:  

  327:                     actions.MoveToElement(selectDeliveryAreaDropDownListFirstSelection).Click().Build().Perform();

  328:                     Thread.Sleep(2000);

  329:  

  330:                     // <button type="submit" value="save store" id="submit_process" name="save_store" class="action submit primary"><span>Continue shopping</span></button>

  331:                     var continueShoppingButton = chromeDriver.FindElement(By.XPath("//button[@id='submit_process']"), 10);

  332:  

  333:                     actions.MoveToElement(continueShoppingButton).Click().Build().Perform();

  334:                     Thread.Sleep(2000);

  335:  

  336:                     var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);

  337:  

  338:                     var content = chromeDriver.PageSource;

  339:  

  340:                     ProductSkuNamePriceStockFromContent(content, out string sku, out string name, out price, out stock);

  341:  

  342:                     product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);

  343:                     product.SiteId = site.Id;

  344:                     //product.Category = product.Category;

  345:                     product.Sku = sku;

  346:                     product.Name = name;

  347:                     product.Url = productUrl;

  348:  

  349:                     //if (content.Contains("product:availability") && content.Contains("in stock"))

  350:                     //{

  351:                     var newAddToCartButtonHasBeenClicked = false;

  352:  

  353:                     exceptionCounter = 0;

  354:                     stock = 0;

  355:                     var stockList = new List<int>();

  356:                     //var list = new List<int>();

  357:  

  358:                     Thread.Sleep(2000);

  359:  

  360:                     do

  361:                     {

  362:                         try

  363:                         {

  364:                             if (!newAddToCartButtonHasBeenClicked)

  365:                             {

  366:                                 // <div data-v-9815bdd8="" class="new-add-to-cart"><div data-v-9815bdd8="" class="mx-0 no-padding"><div data-v-9815bdd8="" class="add-to-cart-btn pl0"><button data-v-9815bdd8="" type="submit" class="btn">Add to cart</button></div></div></div>

  367:                                 var newAddToCartButton = chromeDriver.FindElement(By.XPath("//div[@class='new-add-to-cart']/div/div/button[@class='btn']"), 10);

  368:  

  369:                                 actions.MoveToElement(newAddToCartButton).Click().Build().Perform();

  370:                                 Thread.Sleep(2000);

  371:  

  372:                                 newAddToCartButtonHasBeenClicked = true;

  373:                             }

  374:                             else

  375:                             {

  376:                                 // <div data-v-7ce7152e="" class="col-4 increase-qty"><button data-v-7ce7152e="" type="submit" class="btn addition"></button></div>

  377:                                 var plusSignButton = chromeDriver.FindElement(By.XPath("//div[@class='col-4 increase-qty']/button[@class='btn addition']"), 10);

  378:  

  379:                                 actions.MoveToElement(plusSignButton).Click().Build().Perform();

  380:                                 Thread.Sleep(2000);

  381:                             }

  382:  

  383:                             // <li data-v-731d3b1c="" class="cart_part"><a data-v-731d3b1c="" href="/checkout/cart" class=""><div data-v-731d3b1c="" class="badge badge-primary position-absolute ml-4 mb-4 rounded-circle text-white bg-standard">1</div>

  384:                             var counterNumberTag = chromeDriver.FindElement(By.ClassName("badge"), 10);

  385:  

  386:                             var text = counterNumberTag.Text;

  387:  

  388:                             if (int.TryParse(text, out stock)) stockList.Add(stock);

  389:                             else if (string.IsNullOrEmpty(text))

  390:                             {

  391:                                 stock = 0;

  392:                                 stockList.Add(stock);

  393:                                 stockList.Add(stock);

  394:                                 stockList.Add(stock);

  395:                             }

  396:                         }

  397:                         catch (Exception e)

  398:                         {

  399:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);

  400:                         }

  401:                     }

  402:                     while (!LastThreeNumbersAreIdentical(stockList) && counter++ < 100);

  403:                     //}

  404:                 }

  405:                 catch (Exception e)

  406:                 {

  407:                     if (e.Message.Contains("no such element: Unable to locate element:")) ;

  408:  

  409:                     if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);

  410:                 }

  411:             }

  412:         }

  413:  

  414:         private static bool LastThreeNumbersAreIdentical(List<int> list)

  415:         {

  416:             var identical = false;

  417:  

  418:             if (list.Count >= 3)

  419:             {

  420:                 var length = list.Count;

  421:  

  422:                 if (list[length - 1] == list[length - 2] && list[length - 1] == list[length - 3]) identical = true;

  423:                 else identical = false;

  424:             }

  425:  

  426:             return identical;

  427:         }

  428:  

  429:         ////////////////////////////////////////////////////////////////////////////

  430:  

  431:         ///// <summary>

  432:         ///

  433:         /// </summary>

  434:         public static void ProductSkuNamePriceStockFromContent(string content, out string sku, out string name, out decimal price, out int stock)

  435:         {

  436:             var match = Regex.Match(content, @"<div class=""uppersec""><div class=""topdetails""><div class=""sku"">(.+?)</div></div> <h5 class=""prodname"">(.+?)</h5> <!----> <div class=""price-grams""><div class=""price""> KD (.+?) </div>", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);

  437:  

  438:             if (match.Success)

  439:             {

  440:                 sku = match.Groups[1].Value;

  441:                 name = match.Groups[2].Value;

  442:                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);

  443:  

  444:                 price = decimal.TryParse(match.Groups[3].Value, out decimal d) ? d : 0;

  445:  

  446:                 //stock = int.Parse(match.Groups[6].Value);

  447:                 stock = Ia.Statistics.Cl.Models.Default.StockValueUnAvailable; //stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;

  448:             }

  449:             else

  450:             {

  451:                 name = string.Empty;

  452:                 sku = string.Empty;

  453:  

  454:                 price = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;

  455:                 stock = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;

  456:             }

  457:         }

  458:  

  459:         /*

  460:         ////////////////////////////////////////////////////////////////////////////

  461: 

  462:         /// <summary>

  463:         ///

  464:         /// </summary>

  465:         public static string ReadUpdateProductListOfCategorySequentially()

  466:         {

  467:             string s;

  468:             List<string> productIdList;

  469: 

  470:             var category = Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList.RandomThenNext();

  471: 

  472:             var site = Ia.Statistics.Cl.Model.Site.SiteFromId(category.SiteId);

  473: 

  474:             var productPriceStockQuantitySoldList = ProductPriceStockListOfCategory(site, category, out string exception);

  475: 

  476:             if (string.IsNullOrEmpty(exception))

  477:             {

  478:                 if (productPriceStockQuantitySoldList.Count > 0)

  479:                 {

  480:                     s = string.Empty;

  481:                     productIdList = new List<string>(); // this is needed because sometimes the parser read a product twice

  482: 

  483:                     foreach (var productPriceStockQuantitySold in productPriceStockQuantitySoldList)

  484:                     {

  485:                         if (!productIdList.Contains(productPriceStockQuantitySold.Product.Id))

  486:                         {

  487:                             if (!string.IsNullOrEmpty(productPriceStockQuantitySold.Product.Sku))

  488:                             {

  489:                                 Ia.Statistics.Cl.Model.Product.Create(productPriceStockQuantitySold.Product);

  490:                                 Ia.Statistics.Cl.Model.ProductPriceSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Price);

  491:                                 Ia.Statistics.Cl.Model.ProductStockSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Stock);

  492: 

  493:                                 s += "Product: " + productPriceStockQuantitySold.Product.Name + "; price: " + productPriceStockQuantitySold.Price + "; stock: " + productPriceStockQuantitySold.Stock + ". ";

  494:                             }

  495:                             else

  496:                             {

  497:                                 s += "Product: " + productPriceStockQuantitySold.Product.Name + ": SKU is null. ";

  498:                             }

  499:                         }

  500:                         else

  501:                         {

  502:                             s += "Product: Product with Id " + productPriceStockQuantitySold.Product.Id + " has already been created. ";

  503:                         }

  504: 

  505:                         productIdList.Add(productPriceStockQuantitySold.Product.Id);

  506:                     }

  507:                 }

  508:                 else s = "Product list is 0. ";

  509: 

  510:                 s = site.Name + ": " + category.Url + ": " + s;

  511:             }

  512:             else s = exception;

  513: 

  514:             return s;

  515:         }

  516: 

  517:         ////////////////////////////////////////////////////////////////////////////

  518: 

  519:         /// <summary>

  520:         ///

  521:         /// </summary>

  522:         public static List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold> ProductPriceStockListOfCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, out string exception)

  523:         {

  524:             int areaId;

  525: 

  526:             var productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold>();

  527: 

  528:             var options = new ChromeOptions();

  529: 

  530:             var chromeDriverService = ChromeDriverService.CreateDefaultService();

  531:             chromeDriverService.HideCommandPromptWindow = true;

  532:             chromeDriverService.SuppressInitialDiagnosticInformation = true;

  533: 

  534:             options.AddArgument("--headless");

  535:             options.AddArgument("--log-level=3");

  536:             options.AddArgument("--disable-extensions");

  537:             options.AddArgument("--ignore-certificate-errors");

  538: 

  539:             /*

  540:             switch (site.Iso)

  541:             {

  542:                 case "KW": areaId = 10; break;

  543:                 case "SA": areaId = 2648; break;

  544:                 case "AE": areaId = 12; break;

  545:                 case "QA": areaId = 6; break;

  546:                 case "BH": areaId = 11; break;

  547:                 default: areaId = 10; break;

  548:             }

  549:             * /

  550: 

  551:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))

  552:             {

  553:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan;

  554: 

  555:                 chromeDriver.Navigate().GoToUrl(category.Url);

  556: 

  557:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("DAB_AREA", areaId.ToString(), "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  558:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("DAB_COUNTRY", site.Iso, "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  559:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("accept_cookies", "accepted", "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  560:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("ajs_anonymous_id", "e7a170bb-807b-4546-8caf-b7cec22f20c5", "", DateTime.UtcNow.AddHours(3).AddYears(1)));

  561: 

  562:                 //chromeDriver.Navigate().GoToUrl(category.Url); // I will access site again with location cookies

  563: 

  564:                 // maximize the browser

  565:                 // chromeDriver.Manage().Window.Maximize();

  566: 

  567:                 // scroll down the webpage

  568:                 var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);

  569:                 javaScriptExecutor.ExecuteScript("window.scrollTo(0, document.body.scrollHeight)");

  570: 

  571:                 var content = chromeDriver.PageSource;

  572: 

  573:                 ProductPriceStockListFromCategoryContent(site, category, content, out productPriceStockQuantitySoldList, out exception);

  574:             }

  575: 

  576:             return productPriceStockQuantitySoldList;

  577:         }

  578: 

  579:         ////////////////////////////////////////////////////////////////////////////

  580: 

  581:         /// <summary>

  582:         ///

  583:         /// </summary>

  584:         public static void ProductPriceStockListFromCategoryContent(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, string content, out List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold> productPriceStockQuantitySoldList, out string exception)

  585:         {

  586:             productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold>();

  587: 

  588:             var matchCollection = Regex.Matches(content, @"<div class=""product-box .+? data-url=""(.+?)"" data-productcode=""(\d+?)"">.+?<input type=""hidden"" id="".+?"" class="".+?"" data-id=""\d+?"" data-name=""(.+?)"" data-price=""(.+?)"" .+?", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);

  589: 

  590:             foreach (Match match in matchCollection)

  591:             {

  592:                 var product = new Ia.Statistics.Cl.Model.Product();

  593:                 var productPriceStockQuantitySold = new Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold();

  594: 

  595:                 var sku = match.Groups[2].Value; // data-productcode

  596: 

  597:                 var name = match.Groups[3].Value; // name

  598:                 name = Ia.Statistics.Cl.Model.Default.NormalizeName(name);

  599: 

  600:                 var url = match.Groups[1].Value; // data-url

  601: 

  602:                 product.Id = Ia.Statistics.Cl.Model.Product.ProductId(site, sku);

  603:                 product.SiteId = site.Id;

  604:                 //product.Category = category;

  605:                 product.Sku = sku;

  606:                 product.Name = name;

  607:                 product.Url = site.BaseUrl + url;

  608: 

  609:                 var price = decimal.TryParse(match.Groups[4].Value, out decimal d) ? d : 0;

  610: 

  611:                 //var stock = int.Parse(match.Groups[6].Value); // buy_limit;

  612:                 var stock = Ia.Statistics.Cl.Model.Default.StockValueUnAvailable; // stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;

  613: 

  614:                 productPriceStockQuantitySold.Product = product;

  615:                 productPriceStockQuantitySold.Price = price;

  616:                 productPriceStockQuantitySold.Stock = stock;

  617: 

  618:                 productPriceStockQuantitySoldList.Add(productPriceStockQuantitySold);

  619:             }

  620: 

  621:             exception = string.Empty;

  622:         }

  623: 

  624:         ////////////////////////////////////////////////////////////////////////////

  625:         ////////////////////////////////////////////////////////////////////////////

  626: 

  627:         /// <summary>

  628:         ///

  629:         /// </summary>

  630:         public static string ProductUrlListOfCategorySequentially()

  631:         {

  632:             Ia.Statistics.Cl.Model.Category category;

  633: 

  634:             /*

  635:             if (categoryToNextUrlDictionary.Count > 0)

  636:             {

  637:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();

  638: 

  639:                 category = categoryToNextUrl.Key;

  640:                 category.Url = categoryToNextUrl.Value;

  641: 

  642:                 categoryToNextUrlDictionary.Remove(category);

  643:             }

  644:             else

  645:             {

  646:             * /

  647:             category = Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList.RandomThenNext();

  648:             //category = (from c in Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList where c.Url.Contains("chocolate-candy.html") select c).SingleOrDefault();

  649:             //}

  650: 

  651:             var list = ProductUrlListOfCategory(category);

  652: 

  653:             if (productUrlList.Count > 0)

  654:             {

  655:                 foreach (var l in list)

  656:                 {

  657:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);

  658:                 }

  659:             }

  660:             else productUrlList = list;

  661: 

  662:             return "Category: " + category.Name + "; product count: " + productUrlList.Count + ". ";

  663:         }

  664:         */

  665:  

  666:         ////////////////////////////////////////////////////////////////////////////

  667:         ////////////////////////////////////////////////////////////////////////////

  668:  

  669:         /// <summary>

  670:         ///

  671:         /// </summary>

  672:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList

  673:         {

  674:             get

  675:             {

  676:                 if (categoryList == null || categoryList.Count == 0)

  677:                 {

  678:                     categoryList = (from c in XDocument.Element("sultan-center").Descendants("category")

  679:                                     where !c.HasElements

  680:                                     select new Ia.Statistics.Cl.Models.Category

  681:                                     {

  682:                                         Id = 0, //Ia.Statistics.Cl.Model.Category.CategoryId(Ia.Statistics.Cl.Model.Site.SultanCenterKuwait, int.Parse(c.Attribute("id").Value)),

  683:                                         SiteId = Ia.Statistics.Cl.Models.Site.SultanCenter.Id,

  684:                                         Url = Ia.Statistics.Cl.Models.Site.SultanCenter.BaseUrl + c.Attribute("url").Value,

  685:                                         Name = c.Attribute("name").Value

  686:                                     }

  687:                                     ).ToList<Ia.Statistics.Cl.Models.Category>();

  688:  

  689:                     categoryList = categoryList.Shuffle().ToList();

  690:                 }

  691:  

  692:                 return categoryList;

  693:             }

  694:         }

  695:  

  696:         ////////////////////////////////////////////////////////////////////////////

  697:  

  698:         /// <summary>

  699:         /// 

  700:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us

  701:         /// 

  702:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".

  703:         /// 2. Add "using System.Reflection".

  704:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.

  705:         /// 

  706:         /// </summary>

  707:  

  708:         private static XDocument XDocument

  709:         {

  710:             get

  711:             {

  712:                 Assembly assembly;

  713:                 StreamReader streamReader;

  714:  

  715:                 xd = null;

  716:                 assembly = Assembly.GetExecutingAssembly();

  717:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.sultan_center.com.default.xml"));

  718:  

  719:                 try

  720:                 {

  721:                     if (streamReader.Peek() != -1)

  722:                     {

  723:                         xd = System.Xml.Linq.XDocument.Load(streamReader);

  724:                     }

  725:                 }

  726:                 catch (Exception)

  727:                 {

  728:                 }

  729:                 finally

  730:                 {

  731:                 }

  732:  

  733:                 return xd;

  734:             }

  735:         }

  736:  

  737:         ////////////////////////////////////////////////////////////////////////////

  738:         ////////////////////////////////////////////////////////////////////////////

  739:     }

  740: }