)>}]
شركة التطبيقات المتكاملة لتصميم وبرمجة البرمجيات الخاصة ش.ش.و.
Integrated Applications Programming Company
Home » Code Library » Default (Ia.Statistics.Cl.Models.SultanCenter)

Public general use code classes and xml files that we've compiled and used over the years:

Structure of the sultan-center.com website.

    1: using System;
    2: using System.Collections.Generic;
    3: using System.IO;
    4: using System.Linq;
    5: using System.Reflection;
    6: using System.Xml.Linq;
    7: using System.Text.RegularExpressions;
    8: using OpenQA.Selenium;
    9: using OpenQA.Selenium.Chrome;
   10: using OpenQA.Selenium.Interactions;
   11: using System.Threading;
   12:  
   13: namespace Ia.Statistics.Cl.Models.SultanCenter
   14: {
   15:     ////////////////////////////////////////////////////////////////////////////
   16:  
   17:     /// <summary publish="true">
   18:     /// Structure of the sultan-center.com website.
   19:     /// </summary>
   20:     /// <remarks> 
   21:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.
   22:     ///
   23:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by
   24:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
   25:     ///
   26:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   27:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
   28:     /// 
   29:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.
   30:     /// 
   31:     /// Copyright notice: This notice may not be removed or altered from any source distribution.
   32:     /// </remarks> 
   33:     public class Default
   34:     {
   35:         private static int updateProductUrlListCounter = 0;
   36:  
   37:         private static XDocument xd;
   38:  
   39:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;
   40:  
   41:         //private static Dictionary<Ia.Statistics.Cl.Model.Category, string> categoryToNextUrlDictionary = new Dictionary<Ia.Statistics.Cl.Model.Category, string>();
   42:  
   43:         private static List<string> productUrlList = new List<string>();
   44:  
   45:         ////////////////////////////////////////////////////////////////////////////
   46:  
   47:         /// <summary>
   48:         /// 
   49:         /// </summary>
   50:         public Default() { }
   51:  
   52:         ////////////////////////////////////////////////////////////////////////////
   53:  
   54:         /// <summary>
   55:         ///
   56:         /// </summary>
   57:         public static void UpdateProductUrlList(Ia.Statistics.Cl.Models.Site site)
   58:         {
   59:             var list = ProductUrlListOfCategorySequentially();
   60:  
   61:             var storedList = Ia.Statistics.Cl.Models.Product.UrlListBySiteId(site.Id);
   62:  
   63:             if (productUrlList.Count > 0)
   64:             {
   65:                 foreach (var l in list)
   66:                 {
   67:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   68:                 }
   69:             }
   70:             else productUrlList = list;
   71:  
   72:             if (productUrlList.Count > 0)
   73:             {
   74:                 foreach (var l in storedList)
   75:                 {
   76:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   77:                 }
   78:             }
   79:             else productUrlList = storedList;
   80:         }
   81:  
   82:         ////////////////////////////////////////////////////////////////////////////
   83:  
   84:         /// <summary>
   85:         ///
   86:         /// </summary>
   87:         private static List<string> ProductUrlListOfCategorySequentially()
   88:         {
   89:             Ia.Statistics.Cl.Models.Category category;
   90:  
   91:             /*
   92:             if (categoryToNextUrlDictionary.Count > 0)
   93:             {
   94:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
   95: 
   96:                 category = categoryToNextUrl.Key;
   97:                 category.Url = categoryToNextUrl.Value;
   98: 
   99:                 categoryToNextUrlDictionary.Remove(category);
  100:             }
  101:             else
  102:             {
  103:             */
  104:             category = Ia.Statistics.Cl.Models.SultanCenter.Default.CategoryList.RandomThenNext();
  105:             //}
  106:  
  107:             var list = ProductUrlListOfCategory(category);
  108:  
  109:             return list;
  110:         }
  111:  
  112:         ////////////////////////////////////////////////////////////////////////////
  113:  
  114:         /// <summary>
  115:         ///
  116:         /// </summary>
  117:         public static List<string> ProductUrlListOfCategory(Ia.Statistics.Cl.Models.Category category)
  118:         {
  119:             var counter = 0;
  120:             var exceptionCounter = 0;
  121:  
  122:             var list = new List<string>();
  123:             var listList = new List<List<string>>();
  124:  
  125:             var options = new ChromeOptions();
  126:  
  127:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  128:             chromeDriverService.HideCommandPromptWindow = true;
  129:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  130:  
  131:             options.AddArgument("--headless");
  132:             options.AddArgument("--log-level=3");
  133:             options.AddArgument("--disable-extensions");
  134:             options.AddArgument("--ignore-certificate-errors");
  135:  
  136:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  137:             {
  138:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  139:  
  140:                 chromeDriver.Navigate().GoToUrl(category.Url);
  141:  
  142:                 // I can't suppress delivery area selection with cookies
  143:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("unbxd.userId", "uid-1717900855992-" + areaId, "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  144:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("unbxd.visit", "first_time", "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  145:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("tsc_session", "eyJpdiI6ImJyaU0rRjdpT0lDU2FVazBwZlowblE9PSIsInZhbHVlIjoiODR4OUUvOVFLL3ZUNFJncTZ0cWN4WTl6YWxtb2RSMStwQWthektlR3NKQm0zbE9XZ0JUMEU3NzB5c1hrdENjcjM2SUxYcFZxaEg3OTNKQXd0MHZqRkF0N2x1Z0hRZDBORXdiUTVWbW9tTXBSR3Q4VkVKNzNwQkNjSUg2VCtlaW0iLCJtYWMiOiI2MTA1MzNmMjcwMWMzZjQ4ZmRlNjU3MjJkMjg4NjM3MjZjNzJlMGIxMjQzMDgxYTAxMmQ0ZGQzOWM5NThkZGZhIiwidGFnIjoiIn0%3D", "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  146:  
  147:                 Thread.Sleep(2000); // important
  148:  
  149:                 try
  150:                 {
  151:                     // <input type="text" placeholder="Search in delivery area" class="delivery-input">
  152:                     var selectDeliveryAreaDropDownList = chromeDriver.FindElement(By.XPath("//input[@class='delivery-input']"), 10);
  153:  
  154:                     Actions actions = new Actions(chromeDriver);
  155:                     actions.MoveToElement(selectDeliveryAreaDropDownList).Click().Build().Perform();
  156:                     Thread.Sleep(2000);
  157:  
  158:                     // <li class="autocomplete-result"> Abdally</li >
  159:                     var selectDeliveryAreaDropDownListFirstSelection = chromeDriver.FindElement(By.XPath("//li[@class='autocomplete-result']"), 10);
  160:  
  161:                     actions.MoveToElement(selectDeliveryAreaDropDownListFirstSelection).Click().Build().Perform();
  162:                     Thread.Sleep(2000);
  163:  
  164:                     // <button type="submit" value="save store" id="submit_process" name="save_store" class="action submit primary"><span>Continue shopping</span></button>
  165:                     var continueShoppingButton = chromeDriver.FindElement(By.XPath("//button[@id='submit_process']"), 10);
  166:  
  167:                     actions.MoveToElement(continueShoppingButton).Click().Build().Perform();
  168:                     Thread.Sleep(2000);
  169:  
  170:                     var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);
  171:  
  172:                     do
  173:                     {
  174:                         javaScriptExecutor.ExecuteScript("window.scrollTo(0, document.body.scrollHeight)");
  175:  
  176:                         var content = chromeDriver.PageSource;
  177:  
  178:                         list = ProductUrlListFromCategoryContent(category, content);
  179:  
  180:                         listList.Add(list);
  181:  
  182:                         try
  183:                         {
  184:                             // enabled: <div class="pagination-box"><ul data-v-82963a40="" class="pagination"><!----> <li data-v-82963a40="" class="disabled"><a data-v-82963a40="" tabindex="-1">&lt;</a></li> <li data-v-82963a40="" class="page-item active"><a data-v-82963a40="" tabindex="0">1</a></li><li data-v-82963a40="" class="page-item"><a data-v-82963a40="" tabindex="0">2</a></li><li data-v-82963a40="" class="page-item"><a data-v-82963a40="" tabindex="0">3</a></li><li data-v-82963a40="" class="page-item disabled"><a data-v-82963a40="" tabindex="0" class="">…</a></li> <li data-v-82963a40="" class=""><a data-v-82963a40="" tabindex="0">&gt;</a></li> <!----></ul></div>
  185:                             // disabled: <div...> ... <li data-v-82963a40="" class="disabled"><a data-v-82963a40="" tabindex="-1">&gt;</a>
  186:                             var nextPageButton = chromeDriver.FindElement(By.XPath("//*[@class='pagination-box']/ul/li[not(@class='disabled')]/a[text()='>']"), 10);
  187:  
  188:                             actions.MoveToElement(nextPageButton).Click().Build().Perform();
  189:                             Thread.Sleep(2000);
  190:                         }
  191:                         catch (Exception e)
  192:                         {
  193:                             if (e.Message.Contains("no such element: Unable to locate element:")) break;
  194:  
  195:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);
  196:                         }
  197:                     }
  198:                     while (counter++ < 100);
  199:                 }
  200:                 catch (Exception e)
  201:                 {
  202:                     if (e.Message.Contains("no such element: Unable to locate element:")) { }
  203:  
  204:                     if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);
  205:                 }
  206:             }
  207:  
  208:             return listList.SelectMany(u => u).ToList();
  209:         }
  210:  
  211:         ////////////////////////////////////////////////////////////////////////////
  212:  
  213:         /// <summary>
  214:         ///
  215:         /// </summary>
  216:         public static List<string> ProductUrlListFromCategoryContent(Ia.Statistics.Cl.Models.Category category, string content)
  217:         {
  218:             string url;
  219:  
  220:             var list = new List<string>();
  221:  
  222:             /*
  223:             // below: check if content has a Next link
  224:             var nextMatch = Regex.Match(content, @"em-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  225: 
  226:             if (nextMatch.Success)
  227:             {
  228:                 var nextUrl = nextMatch.Groups[1].Value;
  229: 
  230:                 categoryToNextUrlDictionary[category] = nextUrl;
  231:             }
  232:             */
  233:  
  234:             var matchCollection = Regex.Matches(content, @"<div class=""proddeal_img""><a href=""(.+?)"">", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  235:  
  236:             foreach (Match match in matchCollection)
  237:             {
  238:                 url = match.Groups[1].Value;
  239:  
  240:                 list.Add(url);
  241:             }
  242:  
  243:             return list;
  244:         }
  245:  
  246:         ////////////////////////////////////////////////////////////////////////////
  247:         ////////////////////////////////////////////////////////////////////////////
  248:  
  249:         /// <summary>
  250:         ///
  251:         /// </summary>
  252:         public static string ReadUpdateProductPriceStockByProductUrlSequentially(Ia.Statistics.Cl.Models.Site site)
  253:         {
  254:             var s = string.Empty;
  255:             var productUrl = string.Empty;
  256:  
  257:             if (productUrlList.Count == 0 || updateProductUrlListCounter++ % Ia.Statistics.Cl.Models.Default.AverageNumberOfProductsPerCategory == 0) Ia.Statistics.Cl.Models.SultanCenter.Default.UpdateProductUrlList(site);
  258:  
  259:             if (productUrlList.Count > 0)
  260:             {
  261:                 productUrl = productUrlList.RandomThenNext();
  262:  
  263:                 ReadProductPriceStockByProductUrl(site, productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock);
  264:  
  265:                 if (!string.IsNullOrEmpty(product.Sku))
  266:                 {
  267:                     Ia.Statistics.Cl.Models.Product.Create(product);
  268:                     Ia.Statistics.Cl.Models.ProductPriceSpot.Create(product, price);
  269:                     Ia.Statistics.Cl.Models.ProductStockSpot.Create(product, stock);
  270:  
  271:                     s = "Product: " + product.Name + "; price: " + price + "; stock: " + stock + ". ";
  272:                 }
  273:                 else s = "Product: " + product.Name + ": SKU is null. ";
  274:             }
  275:             else s = "Product list is empty. ";
  276:  
  277:             s = site.Name + ": " + productUrl + ": " + s;
  278:  
  279:             return s;
  280:         }
  281:  
  282:         ////////////////////////////////////////////////////////////////////////////
  283:  
  284:         /// <summary>
  285:         ///
  286:         /// </summary>
  287:         public static void ReadProductPriceStockByProductUrl(Ia.Statistics.Cl.Models.Site site, string productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock)
  288:         {
  289:             var counter = 0;
  290:             var exceptionCounter = 0;
  291:  
  292:             var options = new ChromeOptions();
  293:  
  294:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  295:             chromeDriverService.HideCommandPromptWindow = true;
  296:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  297:  
  298:             options.AddArgument("--headless");
  299:             options.AddArgument("--log-level=3");
  300:             options.AddArgument("--disable-extensions");
  301:             options.AddArgument("--ignore-certificate-errors");
  302:  
  303:             price = stock = 0;
  304:  
  305:             product = new Ia.Statistics.Cl.Models.Product();
  306:  
  307:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  308:             {
  309:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  310:  
  311:                 chromeDriver.Navigate().GoToUrl(productUrl);
  312:  
  313:                 Thread.Sleep(2000); // important
  314:  
  315:                 try
  316:                 {
  317:                     // <input type="text" placeholder="Search in delivery area" class="delivery-input">
  318:                     var selectDeliveryAreaDropDownList = chromeDriver.FindElement(By.XPath("//input[@class='delivery-input']"), 10);
  319:  
  320:                     Actions actions = new Actions(chromeDriver);
  321:                     actions.MoveToElement(selectDeliveryAreaDropDownList).Click().Build().Perform();
  322:                     Thread.Sleep(2000);
  323:  
  324:                     // <li class="autocomplete-result"> Abdally</li >
  325:                     var selectDeliveryAreaDropDownListFirstSelection = chromeDriver.FindElement(By.XPath("//li[@class='autocomplete-result']"), 10);
  326:  
  327:                     actions.MoveToElement(selectDeliveryAreaDropDownListFirstSelection).Click().Build().Perform();
  328:                     Thread.Sleep(2000);
  329:  
  330:                     // <button type="submit" value="save store" id="submit_process" name="save_store" class="action submit primary"><span>Continue shopping</span></button>
  331:                     var continueShoppingButton = chromeDriver.FindElement(By.XPath("//button[@id='submit_process']"), 10);
  332:  
  333:                     actions.MoveToElement(continueShoppingButton).Click().Build().Perform();
  334:                     Thread.Sleep(2000);
  335:  
  336:                     var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);
  337:  
  338:                     var content = chromeDriver.PageSource;
  339:  
  340:                     ProductSkuNamePriceStockFromContent(content, out string sku, out string name, out price, out stock);
  341:  
  342:                     product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);
  343:                     product.SiteId = site.Id;
  344:                     //product.Category = product.Category;
  345:                     product.Sku = sku;
  346:                     product.Name = name;
  347:                     product.Url = productUrl;
  348:  
  349:                     //if (content.Contains("product:availability") && content.Contains("in stock"))
  350:                     //{
  351:                     var newAddToCartButtonHasBeenClicked = false;
  352:  
  353:                     exceptionCounter = 0;
  354:                     stock = 0;
  355:                     var stockList = new List<int>();
  356:                     //var list = new List<int>();
  357:  
  358:                     Thread.Sleep(2000);
  359:  
  360:                     do
  361:                     {
  362:                         try
  363:                         {
  364:                             if (!newAddToCartButtonHasBeenClicked)
  365:                             {
  366:                                 // <div data-v-9815bdd8="" class="new-add-to-cart"><div data-v-9815bdd8="" class="mx-0 no-padding"><div data-v-9815bdd8="" class="add-to-cart-btn pl0"><button data-v-9815bdd8="" type="submit" class="btn">Add to cart</button></div></div></div>
  367:                                 var newAddToCartButton = chromeDriver.FindElement(By.XPath("//div[@class='new-add-to-cart']/div/div/button[@class='btn']"), 10);
  368:  
  369:                                 actions.MoveToElement(newAddToCartButton).Click().Build().Perform();
  370:                                 Thread.Sleep(2000);
  371:  
  372:                                 newAddToCartButtonHasBeenClicked = true;
  373:                             }
  374:                             else
  375:                             {
  376:                                 // <div data-v-7ce7152e="" class="col-4 increase-qty"><button data-v-7ce7152e="" type="submit" class="btn addition"></button></div>
  377:                                 var plusSignButton = chromeDriver.FindElement(By.XPath("//div[@class='col-4 increase-qty']/button[@class='btn addition']"), 10);
  378:  
  379:                                 actions.MoveToElement(plusSignButton).Click().Build().Perform();
  380:                                 Thread.Sleep(2000);
  381:                             }
  382:  
  383:                             // <li data-v-731d3b1c="" class="cart_part"><a data-v-731d3b1c="" href="/checkout/cart" class=""><div data-v-731d3b1c="" class="badge badge-primary position-absolute ml-4 mb-4 rounded-circle text-white bg-standard">1</div>
  384:                             var counterNumberTag = chromeDriver.FindElement(By.ClassName("badge"), 10);
  385:  
  386:                             var text = counterNumberTag.Text;
  387:  
  388:                             if (int.TryParse(text, out stock)) stockList.Add(stock);
  389:                             else if (string.IsNullOrEmpty(text))
  390:                             {
  391:                                 stock = 0;
  392:                                 stockList.Add(stock);
  393:                                 stockList.Add(stock);
  394:                                 stockList.Add(stock);
  395:                             }
  396:                         }
  397:                         catch (Exception e)
  398:                         {
  399:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);
  400:                         }
  401:                     }
  402:                     while (!LastThreeNumbersAreIdentical(stockList) && counter++ < 100);
  403:                     //}
  404:                 }
  405:                 catch (Exception e)
  406:                 {
  407:                     if (e.Message.Contains("no such element: Unable to locate element:")) ;
  408:  
  409:                     if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);
  410:                 }
  411:             }
  412:         }
  413:  
  414:         private static bool LastThreeNumbersAreIdentical(List<int> list)
  415:         {
  416:             var identical = false;
  417:  
  418:             if (list.Count >= 3)
  419:             {
  420:                 var length = list.Count;
  421:  
  422:                 if (list[length - 1] == list[length - 2] && list[length - 1] == list[length - 3]) identical = true;
  423:                 else identical = false;
  424:             }
  425:  
  426:             return identical;
  427:         }
  428:  
  429:         ////////////////////////////////////////////////////////////////////////////
  430:  
  431:         ///// <summary>
  432:         ///
  433:         /// </summary>
  434:         public static void ProductSkuNamePriceStockFromContent(string content, out string sku, out string name, out decimal price, out int stock)
  435:         {
  436:             var match = Regex.Match(content, @"<div class=""uppersec""><div class=""topdetails""><div class=""sku"">(.+?)</div></div> <h5 class=""prodname"">(.+?)</h5> <!----> <div class=""price-grams""><div class=""price""> KD (.+?) </div>", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  437:  
  438:             if (match.Success)
  439:             {
  440:                 sku = match.Groups[1].Value;
  441:                 name = match.Groups[2].Value;
  442:                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);
  443:  
  444:                 price = decimal.TryParse(match.Groups[3].Value, out decimal d) ? d : 0;
  445:  
  446:                 //stock = int.Parse(match.Groups[6].Value);
  447:                 stock = Ia.Statistics.Cl.Models.Default.StockValueUnAvailable; //stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  448:             }
  449:             else
  450:             {
  451:                 name = string.Empty;
  452:                 sku = string.Empty;
  453:  
  454:                 price = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  455:                 stock = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  456:             }
  457:         }
  458:  
  459:         /*
  460:         ////////////////////////////////////////////////////////////////////////////
  461: 
  462:         /// <summary>
  463:         ///
  464:         /// </summary>
  465:         public static string ReadUpdateProductListOfCategorySequentially()
  466:         {
  467:             string s;
  468:             List<string> productIdList;
  469: 
  470:             var category = Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList.RandomThenNext();
  471: 
  472:             var site = Ia.Statistics.Cl.Model.Site.SiteFromId(category.SiteId);
  473: 
  474:             var productPriceStockQuantitySoldList = ProductPriceStockListOfCategory(site, category, out string exception);
  475: 
  476:             if (string.IsNullOrEmpty(exception))
  477:             {
  478:                 if (productPriceStockQuantitySoldList.Count > 0)
  479:                 {
  480:                     s = string.Empty;
  481:                     productIdList = new List<string>(); // this is needed because sometimes the parser read a product twice
  482: 
  483:                     foreach (var productPriceStockQuantitySold in productPriceStockQuantitySoldList)
  484:                     {
  485:                         if (!productIdList.Contains(productPriceStockQuantitySold.Product.Id))
  486:                         {
  487:                             if (!string.IsNullOrEmpty(productPriceStockQuantitySold.Product.Sku))
  488:                             {
  489:                                 Ia.Statistics.Cl.Model.Product.Create(productPriceStockQuantitySold.Product);
  490:                                 Ia.Statistics.Cl.Model.ProductPriceSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Price);
  491:                                 Ia.Statistics.Cl.Model.ProductStockSpot.Create(productPriceStockQuantitySold.Product, productPriceStockQuantitySold.Stock);
  492: 
  493:                                 s += "Product: " + productPriceStockQuantitySold.Product.Name + "; price: " + productPriceStockQuantitySold.Price + "; stock: " + productPriceStockQuantitySold.Stock + ". ";
  494:                             }
  495:                             else
  496:                             {
  497:                                 s += "Product: " + productPriceStockQuantitySold.Product.Name + ": SKU is null. ";
  498:                             }
  499:                         }
  500:                         else
  501:                         {
  502:                             s += "Product: Product with Id " + productPriceStockQuantitySold.Product.Id + " has already been created. ";
  503:                         }
  504: 
  505:                         productIdList.Add(productPriceStockQuantitySold.Product.Id);
  506:                     }
  507:                 }
  508:                 else s = "Product list is 0. ";
  509: 
  510:                 s = site.Name + ": " + category.Url + ": " + s;
  511:             }
  512:             else s = exception;
  513: 
  514:             return s;
  515:         }
  516: 
  517:         ////////////////////////////////////////////////////////////////////////////
  518: 
  519:         /// <summary>
  520:         ///
  521:         /// </summary>
  522:         public static List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold> ProductPriceStockListOfCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, out string exception)
  523:         {
  524:             int areaId;
  525: 
  526:             var productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold>();
  527: 
  528:             var options = new ChromeOptions();
  529: 
  530:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  531:             chromeDriverService.HideCommandPromptWindow = true;
  532:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  533: 
  534:             options.AddArgument("--headless");
  535:             options.AddArgument("--log-level=3");
  536:             options.AddArgument("--disable-extensions");
  537:             options.AddArgument("--ignore-certificate-errors");
  538: 
  539:             /*
  540:             switch (site.Iso)
  541:             {
  542:                 case "KW": areaId = 10; break;
  543:                 case "SA": areaId = 2648; break;
  544:                 case "AE": areaId = 12; break;
  545:                 case "QA": areaId = 6; break;
  546:                 case "BH": areaId = 11; break;
  547:                 default: areaId = 10; break;
  548:             }
  549:             * /
  550: 
  551:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  552:             {
  553:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan;
  554: 
  555:                 chromeDriver.Navigate().GoToUrl(category.Url);
  556: 
  557:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("DAB_AREA", areaId.ToString(), "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  558:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("DAB_COUNTRY", site.Iso, "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  559:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("accept_cookies", "accepted", "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  560:                 //chromeDriver.Manage().Cookies.AddCookie(new OpenQA.Selenium.Cookie("ajs_anonymous_id", "e7a170bb-807b-4546-8caf-b7cec22f20c5", "", DateTime.UtcNow.AddHours(3).AddYears(1)));
  561: 
  562:                 //chromeDriver.Navigate().GoToUrl(category.Url); // I will access site again with location cookies
  563: 
  564:                 // maximize the browser
  565:                 // chromeDriver.Manage().Window.Maximize();
  566: 
  567:                 // scroll down the webpage
  568:                 var javaScriptExecutor = ((IJavaScriptExecutor)chromeDriver);
  569:                 javaScriptExecutor.ExecuteScript("window.scrollTo(0, document.body.scrollHeight)");
  570: 
  571:                 var content = chromeDriver.PageSource;
  572: 
  573:                 ProductPriceStockListFromCategoryContent(site, category, content, out productPriceStockQuantitySoldList, out exception);
  574:             }
  575: 
  576:             return productPriceStockQuantitySoldList;
  577:         }
  578: 
  579:         ////////////////////////////////////////////////////////////////////////////
  580: 
  581:         /// <summary>
  582:         ///
  583:         /// </summary>
  584:         public static void ProductPriceStockListFromCategoryContent(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, string content, out List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold> productPriceStockQuantitySoldList, out string exception)
  585:         {
  586:             productPriceStockQuantitySoldList = new List<Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold>();
  587: 
  588:             var matchCollection = Regex.Matches(content, @"<div class=""product-box .+? data-url=""(.+?)"" data-productcode=""(\d+?)"">.+?<input type=""hidden"" id="".+?"" class="".+?"" data-id=""\d+?"" data-name=""(.+?)"" data-price=""(.+?)"" .+?", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  589: 
  590:             foreach (Match match in matchCollection)
  591:             {
  592:                 var product = new Ia.Statistics.Cl.Model.Product();
  593:                 var productPriceStockQuantitySold = new Ia.Statistics.Cl.Model.ProductPriceStockQuantitySold();
  594: 
  595:                 var sku = match.Groups[2].Value; // data-productcode
  596: 
  597:                 var name = match.Groups[3].Value; // name
  598:                 name = Ia.Statistics.Cl.Model.Default.NormalizeName(name);
  599: 
  600:                 var url = match.Groups[1].Value; // data-url
  601: 
  602:                 product.Id = Ia.Statistics.Cl.Model.Product.ProductId(site, sku);
  603:                 product.SiteId = site.Id;
  604:                 //product.Category = category;
  605:                 product.Sku = sku;
  606:                 product.Name = name;
  607:                 product.Url = site.BaseUrl + url;
  608: 
  609:                 var price = decimal.TryParse(match.Groups[4].Value, out decimal d) ? d : 0;
  610: 
  611:                 //var stock = int.Parse(match.Groups[6].Value); // buy_limit;
  612:                 var stock = Ia.Statistics.Cl.Model.Default.StockValueUnAvailable; // stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  613: 
  614:                 productPriceStockQuantitySold.Product = product;
  615:                 productPriceStockQuantitySold.Price = price;
  616:                 productPriceStockQuantitySold.Stock = stock;
  617: 
  618:                 productPriceStockQuantitySoldList.Add(productPriceStockQuantitySold);
  619:             }
  620: 
  621:             exception = string.Empty;
  622:         }
  623: 
  624:         ////////////////////////////////////////////////////////////////////////////
  625:         ////////////////////////////////////////////////////////////////////////////
  626: 
  627:         /// <summary>
  628:         ///
  629:         /// </summary>
  630:         public static string ProductUrlListOfCategorySequentially()
  631:         {
  632:             Ia.Statistics.Cl.Model.Category category;
  633: 
  634:             /*
  635:             if (categoryToNextUrlDictionary.Count > 0)
  636:             {
  637:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
  638: 
  639:                 category = categoryToNextUrl.Key;
  640:                 category.Url = categoryToNextUrl.Value;
  641: 
  642:                 categoryToNextUrlDictionary.Remove(category);
  643:             }
  644:             else
  645:             {
  646:             * /
  647:             category = Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList.RandomThenNext();
  648:             //category = (from c in Ia.Statistics.Cl.Model.SultanCenter.Default.CategoryList where c.Url.Contains("chocolate-candy.html") select c).SingleOrDefault();
  649:             //}
  650: 
  651:             var list = ProductUrlListOfCategory(category);
  652: 
  653:             if (productUrlList.Count > 0)
  654:             {
  655:                 foreach (var l in list)
  656:                 {
  657:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
  658:                 }
  659:             }
  660:             else productUrlList = list;
  661: 
  662:             return "Category: " + category.Name + "; product count: " + productUrlList.Count + ". ";
  663:         }
  664:         */
  665:  
  666:         ////////////////////////////////////////////////////////////////////////////
  667:         ////////////////////////////////////////////////////////////////////////////
  668:  
  669:         /// <summary>
  670:         ///
  671:         /// </summary>
  672:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList
  673:         {
  674:             get
  675:             {
  676:                 if (categoryList == null || categoryList.Count == 0)
  677:                 {
  678:                     categoryList = (from c in XDocument.Element("sultan-center").Descendants("category")
  679:                                     where !c.HasElements
  680:                                     select new Ia.Statistics.Cl.Models.Category
  681:                                     {
  682:                                         Id = 0, //Ia.Statistics.Cl.Model.Category.CategoryId(Ia.Statistics.Cl.Model.Site.SultanCenterKuwait, int.Parse(c.Attribute("id").Value)),
  683:                                         SiteId = Ia.Statistics.Cl.Models.Site.SultanCenter.Id,
  684:                                         Url = Ia.Statistics.Cl.Models.Site.SultanCenter.BaseUrl + c.Attribute("url").Value,
  685:                                         Name = c.Attribute("name").Value
  686:                                     }
  687:                                     ).ToList<Ia.Statistics.Cl.Models.Category>();
  688:  
  689:                     categoryList = categoryList.Shuffle().ToList();
  690:                 }
  691:  
  692:                 return categoryList;
  693:             }
  694:         }
  695:  
  696:         ////////////////////////////////////////////////////////////////////////////
  697:  
  698:         /// <summary>
  699:         /// 
  700:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us
  701:         /// 
  702:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".
  703:         /// 2. Add "using System.Reflection".
  704:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.
  705:         /// 
  706:         /// </summary>
  707:  
  708:         private static XDocument XDocument
  709:         {
  710:             get
  711:             {
  712:                 Assembly assembly;
  713:                 StreamReader streamReader;
  714:  
  715:                 xd = null;
  716:                 assembly = Assembly.GetExecutingAssembly();
  717:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.sultan_center.com.default.xml"));
  718:  
  719:                 try
  720:                 {
  721:                     if (streamReader.Peek() != -1)
  722:                     {
  723:                         xd = System.Xml.Linq.XDocument.Load(streamReader);
  724:                     }
  725:                 }
  726:                 catch (Exception)
  727:                 {
  728:                 }
  729:                 finally
  730:                 {
  731:                 }
  732:  
  733:                 return xd;
  734:             }
  735:         }
  736:  
  737:         ////////////////////////////////////////////////////////////////////////////
  738:         ////////////////////////////////////////////////////////////////////////////
  739:     }
  740: }