抓取浏览器

          1
          const puppeteer = require('puppeteer-core');  
        

          2
          const AUTH = 'PROXY-FULL-ACCOUNT:PASSWORD';  
        

          3
          const WS_ENDPOINT = 'wss://${AUTH}@ws-browser.thordata.com';  
        

          4
            
        

          5
          (async () => {
        

          6
              console.log('Connecting to Scraping Browser...');  
        

          7
              const browser = await puppeteer.connect({  
        

          8
                  browserWSEndpoint: SBR_WS_ENDPOINT,
        

          9
                  defaultViewport: {width: 1920, height: 1080}  
        

          10
             });  
        

          11
              try {  
        

          12
                  console.log('Connected! Navigating to Target URL');  
        

          13
                  const page = await browser.newPage();  
        

          14
                  
        

          15
                  await page.goto('https://example.com', { timeout: 2 * 60 * 1000 });  
        

          16
          
        

          17
                  //1.Screenshot
        

          18
                  console.log('Screenshot to page.png');  
        

          19
                  await page.screenshot({ path: 'remote_screenshot.png' }); 
        

          20
                  console.log('Screenshot be saved');  
        

          21
          
        

          22
                  //2.Get content
        

          23
                  console.log('Get page content...');  
        

          24
                  const html = await page.content();  
        

          25
                  console.log("source Htmml: ", html)  
        

          26
          
        

          27
              } finally {  
        

          28
                  // In order to better use the Scraping browser, be sure to close the browser after the script is executed
        

          29
                  await browser.close();  
        

          30
             }  
        

          31
          })();
        

          1
          import asyncio
        

          2
          from playwright.async_api import async_playwright
        

          3
          AUTH = 'PROXY-FULL-ACCOUNT:PASSWORD'  
        

          4
          WS_ENDPOINT = 'wss://${AUTH}@ws-browser.thordata.com'
        

          5
          
        

          6
          async def run(driver):
        

          7
          
        

          8
              print('Connecting to Scraping Browser...')
        

          9
              browser = await driver.chromium.connect_over_cdp(WS_ENDPOINT)
        

          10
              try:
        

          11
                  # Create a new page
        

          12
                  print('Creating a new page...')
        

          13
                  page = await browser.new_page()
        

          14
                  
        

          15
                  # Navigate to Target URL
        

          16
                  print('Navigating to Target URL...')
        

          17
                  await page.goto('https://www.example.com')
        

          18
          
        

          19
                  # Get page screenshot
        

          20
                  print('Taking page screenshot...')
        

          21
                  await page.screenshot(path='./screenshot.png', full_page=True)
        

          22
                  print('Screenshot saved successfully')
        

          23
          
        

          24
                  # Get page content
        

          25
                  html = await page.content()
        

          26
                  print(html)
        

          27
              finally:
        

          28
                  await browser.close()
        

          29
          
        

          30
          async def main():
        

          31
              async with async_playwright() as playwright:
        

          32
                  await run(playwright)
        

          33
          
        

          34
          if __name__ == '__main__':
        

          35
              asyncio.run(main())
        

          36
                              
        

          37
                          
        

          38
                        
        

          1
          const playwright = require('playwright');
        

          2
          const AUTH = 'PROXY-FULL-ACCOUNT:PASSWORD';  
        

          3
          const WS_ENDPOINT = 'wss://${AUTH}@ws-browser.thordata.com'
        

          4
            
        

          5
          
        

          6
          async function main() {  
        

          7
              console.log('Connecting to Scraping Browser...');  
        

          8
              const browser = await playwright.chromium.connectOverCDP(WS_ENDPOINT);  
        

          9
              try {  
        

          10
                  // Create a new page
        

          11
                  console.log('Creating a new page...');  
        

          12
                  const page = await browser.newPage();
        

          13
          
        

          14
                  // Navigate to Target URL
        

          15
                  await page.goto('https://www.example.com', { timeout: 2 * 60 * 1000 });
        

          16
          
        

          17
                  // Take screenshot
        

          18
                  console.log('Taking screenshot to screenshot.png');  
        

          19
                  await page.screenshot({ path: './screenshot.png', fullPage: true });  
        

          20
          
        

          21
                  // Get page content
        

          22
                  console.log('Scraping page content...');  
        

          23
                  const html = await page.content();  
        

          24
                  console.log(html);  
        

          25
                  
        

          26
              } finally {  
        

          27
                  await browser.close();  
        

          28
             }  
        

          29
          }  
        

          30
            
        

          31
          if (require.main === module) {  
        

          32
              main().catch(err => {  
        

          33
                  console.error(err.stack || err);  
        

          34
                  process.exit(1);  
        

          35
             });  
        

          36
          }
        

          37
                              
        

          38
                          
        

          39
                        
        

          1
          from selenium.webdriver import Remote, ChromeOptions  
        

          2
          from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection  
        

          3
          from selenium.webdriver.common.by import By  
        

          4
          
        

          5
          # Enter your credentials - the zone name and password  
        

          6
          AUTH = 'PROXY-FULL-ACCOUNT:PASSWORD'  
        

          7
          REMOTE_WEBDRIVER = f'https://{AUTH}@hs-browser.thordata.com'  
        

          8
            
        

          9
          def main():  
        

          10
              print('Connecting to Scraping Browser...')  
        

          11
              sbr_connection = ChromiumRemoteConnection(REMOTE_WEBDRIVER, 'goog', 'chrome')  
        

          12
              with Remote(sbr_connection, options=ChromeOptions()) as driver:  
        

          13
          
        

          14
                  # get target URL
        

          15
                  print('Connected! Navigating to target ...')  
        

          16
                  driver.get('https://example.com') 
        

          17
          
        

          18
                  # screenshot 
        

          19
                  print('screenshot to png')  
        

          20
                  driver.get_screenshot_as_file('./remote_page.png')  
        

          21
          
        

          22
                  # html content
        

          23
                  print('Get page content...')  
        

          24
                  html = driver.page_source  
        

          25
                  print(html)  
        

          26
            
        

          27
          if __name__ == '__main__':  
        

          28
             main()  
        

          29