Just wondering if anyone can porvide any insight to a problem that I am having. I am trying to scrape a site using selenium and chromedriver. The site seems to always detect the chromedriver browswer as a bot. here is my configuration, how can I avoid this detection?
Selenium build info
Build info: version: '3.8.1', revision: '6e95a6684b', time: '2017-12-01T19:05:32.194Z' System info: host: 'localhost', ip: '127.0.0.1', os.name: 'Linux', os.arch: 'amd64', os.version: '3.13.0-108-generic', java.version: '1.8.0_151'
I am using ChromeDriver 2.33.506092. Any help is appreciated
<?php // An example of using php-webdriver. namespace Facebook\WebDriver; use Facebook\WebDriver\Chrome\ChromeOptions; use Facebook\WebDriver\Remote\DesiredCapabilities; use Facebook\WebDriver\Remote\RemoteWebDriver; require_once dirname(__FILE__) .'/vendor/autoload.php'; $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; $host = 'http://localhost:4321/wd/hub'; $url = $host; $options = new ChromeOptions(); // Setting the binary is optional. //$options->setBinary('/bin/chromedriver'); // Available options: // http://peter.sh/experiments/chromium-command-line-switches/ $options->addArguments(array( //'--window-size=571,428', // '--start-maximized', // '--headless', "--disable-gpu", "--window-size=1920,1080", "--no-sandbox", '--user-agent=' . $userAgent, "disable-infobars" )); // Setting extensions is also optional //$options->addExtensions(array( // '/path/to/chrome/extension1.crx', // '/path/to/chrome/extension2.crx', //)); $caps = DesiredCapabilities::chrome(); $caps->setPlatform('WINDOWS'); $caps->setCapability(ChromeOptions::CAPABILITY, $options); $driver = RemoteWebDriver::create($url, $caps,5000); $driver->manage()->deleteAllCookies(); $url2 = 'https://www.realtor.ca/Residential/Map.aspx#CultureId=1&ApplicationId=1&RecordsPerPage=9&MaximumResults=9&PropertySearchTypeId=1&TransactionTypeId=2&StoreyRange=0-0&BedRange=0-0&BathRange=0-0&LongitudeMin=-156.0498046875&LongitudeMax=-16.9189453125&LatitudeMin=34.88592973581311&LatitudeMax=72.71190267150946&SortOrder=A&SortBy=1&viewState=m&ZoomLevel=4&PropertyTypeGroupID=1'; $driver->get($url2); sleep(20) ; $cookies = $driver->manage()->getCookies(); 