Peter Hansen princePeterHansen
princePeterHansen
/ books_to_scrape_full_json_like_response.json
Last active
February 18, 2020 22:17
Response with JSON like format
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "selector": "article.product_pod", | |
| "get": "json", | |
| "extract": [ | |
| { | |
| "selector": "h3", | |
| "get": "text", | |
| "as": "title" | |
| }, |
princePeterHansen
/ books_to_scrape_basic_request.json
Created
February 18, 2020 21:10
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "selector": "article.product_pod h3", | |
| "get": "text" | |
| }, | |
| { | |
| "selector": "article.product_pod h3 a", | |
| "get": "attribute", | |
| "attribute": "href" | |
| } |
princePeterHansen
/ books_to_scrape_basic_response.json
Created
February 18, 2020 21:06
Basic response from books to scrape
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "selector": "article.product_pod h3", | |
| "get": "text", | |
| "data": [ | |
| "A Light in the ...", | |
| "Tipping the Velvet", | |
| "Soumission", | |
| "Sharp Objects", | |
| "Sapiens: A Brief History ...", |
princePeterHansen
/ book_element_markup.html
Created
February 18, 2020 20:42
Example of book element markup
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <article class="product_pod"> | |
| <div class="image_container"> | |
| <a href="catalogue/a-light-in-the-attic_1000/index.html"><img | |
| src="media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg" alt="A Light in the Attic" | |
| class="thumbnail" /></a> | |
| </div> | |
| <h3> | |
| <a href="catalogue/a-light-in-the-attic_1000/index.html" title="A Light in the Attic">A Light in the ...</a> | |
| </h3> |
princePeterHansen
/ web_scraping_response_example.json
Created
February 18, 2020 20:19
An example web scraping response from proxybot.io
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "data": [ | |
| { | |
| "title": "Our Band Could Be ...", | |
| "price": "£57.25", | |
| "image": "media/cache/54/60/54607fe8945897cdcced0044103b10b6.jpg", | |
| "link": "catalogue/our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html" | |
| }, | |
| { | |
| "title": "Libertarianism for Beginners", |
princePeterHansen
/ basic_web_scraping_request.json
Last active
February 18, 2020 20:13
Basic web scraping request with Proxybot.io
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "selector": "#someId .someClass a", | |
| "get": "text" | |
| } | |
| ] |
princePeterHansen
/ scraping_with_proxybot_request_body.json
Created
February 17, 2020 22:14
Scraping http://books.toscrape.com with proxybot service
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "selector": "article.product_pod", | |
| "get": "json", | |
| "extract": [ | |
| { | |
| "selector": "h3", | |
| "get": "text", | |
| "as": "title" | |
| }, |
princePeterHansen
/ puppeter_getting_page_html.js
Created
February 10, 2020 21:33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const puppeteer = require('puppeteer'); | |
| const express = require('express'); | |
| const app = express(); | |
| const port = 3000; | |
| app.get('/', async (req, res) => { | |
| const {url} = req.query; | |
| if(!url) { | |
| res.status(400).send("Bad request: 'url' param is missing!"); |
princePeterHansen
/ puppeteer_with_proxybot.js
Created
February 8, 2020 21:53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const puppeteer = require('puppeteer'); | |
| async function run() { | |
| const browser = await puppeteer.launch({ headless: false }); | |
| const page = await browser.newPage(); | |
| const proxy = 'https://proxybot.io/api/v1/API_KEY?url='; | |
| const url = 'https://whatismyipaddress.com/'; | |
| const pageUrl = proxy + url; |
princePeterHansen
/ puppeteer_with_proxy.js
Last active
February 28, 2025 10:49
Using a proxy in puppeteer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const puppeteer = require('puppeteer'); | |
| async function run() { | |
| const browser = await puppeteer.launch({ | |
| headless: false, | |
| args: [ '--proxy-server=200.73.128.156:3128' ] | |
| }); | |
| const page = await browser.newPage(); | |
| const pageUrl = 'https://whatismyipaddress.com/'; |
NewerOlder