Storing the data into a csv file sounds like a good solution and this is where csv-writer will help us out. We need to specify the path and name of the csv file in the path parameter and in the header parameter, and we will have to specify a list of objects. Each object will represent a column of our csv file. The title property of these objects represents the title of each column while the id property needs to match the properties of the objects in our list of leads.
Now, if we wrap the whole code with an async function and add a loop to scrape the first 5 pages for businesses, the code should look like this:
const {JSDOM} = require("jsdom");
const got = require("got");
(async () => {
const leads = []
const nrPages = 5
for (let page = 1; page <= nrPages; page++) {
const url = "https://www.yell.com/ucs/UcsSearchAction.do?keywords=restaurants%26location=United+Kingdom%26scrambleSeed=1024089043%26pageNum=" + page
const params = {
api_key: "XXX",
url: url
}
const response = await got('https://api.webscrapingapi.com/v1', {searchParams: params})
const {document} = new JSDOM(response.body).window
const relatedElements = document.querySelectorAll('.businessCapsule--mainRow')
if (relatedElements) {
relatedElements.forEach(el => {
const businessName = el.querySelector('.businessCapsule--name')
const businessRatingAverage = el.querySelector('.starRating--average')
const businessRatingTotal = el.querySelector('.starRating--total span')
const businessContact = el.querySelector('.business--telephoneNumber')
leads.push({
businessName: businessName ? businessName.innerHTML : 'No business name',
businessRatingAverage: businessRatingAverage ? businessRatingAverage.innerHTML : 'No ratings',
businessRatingTotal: businessRatingTotal ? businessRatingTotal.innerHTML : 'No ratings',
businessContact: businessContact ? businessContact.innerHTML : 'No phone number'
})
})
}
}
const csvWriter = require('csv-writer').createObjectCsvWriter({
path: 'leads.csv',
header: [
{id: 'businessName', title: 'Business Name'},
{id: 'businessRatingAverage', title: 'Business Average Rating'},
{id: 'businessRatingTotal', title: 'Business No. of Ratings'},
{id: 'businessContact', title: 'Business Phone Number'},
]
})
csvWriter.writeRecords(leads).then(() => console.log('Success!!'))
})();
