Puppeteer is a NodeJS library that gives us control over headless Chrome APIs. with Puppeteer, it is much easier to automate UI tests of your website.


Simple to create a new js script (website.js) will easily way to scrape all websites by the puppeteer. 

const puppeteer = require("puppeteer");

// we're using async/await - so we need an async function, that we can run
const scrape = async () => {
  // open the browser and prepare a page
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  // open the page to scrape
  await page.goto("https://github.com/marketplace");

  // execute the JS in the context of the page to get all the links
  const links = await page.evaluate(() => 
    // let's just get all links and create an array from the resulting NodeList
    Array.from(document.querySelectorAll("a")).map(anchor => [anchor.href, anchor.textContent.replace(/\n/, '').trim().replace(/\s\s+/g, ' ')])
  );

  // output all the links
  console.log(links);

  // close the browser 
  await browser.close();
};

// scrape the async function
scrape();


The results are as follows:

❯ node websites.js
[
  [
    'https://github.com/marketplace#start-of-content',
    'Skip to content'
  ],
  [ 'https://github.com/', '' ],
  [
    'https://github.com/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2Fmarketplace&source=header',
    'Sign up'
  ],
  [ 'https://github.com/features', 'Features →' ],
  [ 'https://github.com/mobile', 'Mobile →' ],
  [ 'https://github.com/features/actions', 'Actions →' ],
  [ 'https://github.com/features/codespaces', 'Codespaces →' ],
  [ 'https://github.com/features/packages', 'Packages →' ],
  [ 'https://github.com/features/security', 'Security →' ],
  [ 'https://github.com/features/code-review/', 'Code review →' ],
  [ 'https://github.com/features/issues/', 'Issues →' ],
  [ 'https://github.com/features/integrations', 'Integrations →' ],
  [ 'https://github.com/sponsors', 'GitHub Sponsors →' ],
  [ 'https://github.com/customer-stories', 'Customer stories→' ],
  [ 'https://github.com/team', 'Team' ],
  [ 'https://github.com/enterprise', 'Enterprise' ],
  [ 'https://github.com/explore', 'Explore GitHub →' ],
  [ 'https://github.com/topics', 'Topics →' ],
  [ 'https://github.com/collections', 'Collections →' ],
  [ 'https://github.com/trending', 'Trending →' ],
  [ 'https://lab.github.com/', 'Learning Lab →' ],
  [ 'https://opensource.guide/', 'Open source guides →' ],
  [ 'https://github.com/readme', 'The ReadME Project →' ],
  [ 'https://github.com/events', 'Events →' ],
  [ 'https://github.community/', 'Community forum →' ],
  [ 'https://education.github.com/', 'GitHub Education →' ],
  [ 'https://stars.github.com/', 'GitHub Stars program →' ],
  [ 'https://github.com/marketplace', 'Marketplace' ],
  [ 'https://github.com/pricing', 'Plans →' ],
  [
    'https://github.com/pricing#feature-comparison',
    'Compare plans →'
  ],
  [ 'https://enterprise.github.com/contact', 'Contact Sales →' ],
  [ 'https://education.github.com/', 'Education →' ],
  [ 'https://github.com/marketplace', 'Search All GitHub ↵ Jump to ↵' ],
  [ 'https://github.com/marketplace', 'Search All GitHub ↵ Jump to ↵' ],
  [ 'https://github.com/marketplace', 'Search All GitHub ↵ Jump to ↵' ],
  [ 'https://github.com/marketplace', 'Search All GitHub ↵ Jump to ↵' ],
  [
    'https://github.com/login?return_to=https%3A%2F%2Fgithub.com%2Fmarketplace',
    'Sign in'
  ],
  [
    'https://github.com/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2Fmarketplace&source=header',
    'Sign up'
  ],
  [
    'https://github.com/marketplace/category/free',
    'Explore free apps'
  ],
  [
    'https://github.com/marketplace?category=&query=&type=&verification=',
    'Best Match'
  ],
  [
    'https://github.com/marketplace?category=&query=sort%3Acreated-desc&type=&verification=',
    'Recently added'
  ],
  [
    'https://github.com/marketplace?category=&query=sort%3Apopularity-desc&type=&verification=',
    'Most installed/starred'
  ],
  [
    'https://github.com/marketplace?category=&query=&type=apps&verification=',
    'Apps'
  ],
  [
    'https://github.com/marketplace?category=&query=&type=actions&verification=',
    'Actions'
  ],
  [
    'https://github.com/marketplace?category=api-management&query=&type=&verification=',
    'API management'
  ],
  [
    'https://github.com/marketplace?category=chat&query=&type=&verification=',
    'Chat'
  ],
  [
    'https://github.com/marketplace?category=code-quality&query=&type=&verification=',
    'Code quality'
  ],
  [
    'https://github.com/marketplace?category=code-review&query=&type=&verification=',
    'Code review'
  ],
  [
    'https://github.com/marketplace?category=continuous-integration&query=&type=&verification=',
    'Continuous integration'
  ],
  [
    'https://github.com/marketplace?category=dependency-management&query=&type=&verification=',
    'Dependency management'
  ],
  [
    'https://github.com/marketplace?category=deployment&query=&type=&verification=',
    'Deployment'
  ],
  [
    'https://github.com/marketplace?category=ides&query=&type=&verification=',
    'IDEs'
  ],
  [
    'https://github.com/marketplace?category=learning&query=&type=&verification=',
    'Learning'
  ],
  [
    'https://github.com/marketplace?category=localization&query=&type=&verification=',
    'Localization'
  ],
  [
    'https://github.com/marketplace?category=mobile&query=&type=&verification=',
    'Mobile'
  ],
  [
    'https://github.com/marketplace?category=monitoring&query=&type=&verification=',
    'Monitoring'
  ],
  [
    'https://github.com/marketplace?category=project-management&query=&type=&verification=',
    'Project management'
  ],
  [
    'https://github.com/marketplace?category=publishing&query=&type=&verification=',
    'Publishing'
  ],
  [
    'https://github.com/marketplace?category=recently-added&query=&type=&verification=',
    'Recently added'
  ],
  [
    'https://github.com/marketplace?category=security&query=&type=&verification=',
    'Security'
  ],
  [
    'https://github.com/marketplace?category=support&query=&type=&verification=',
    'Support'
  ],
  [
    'https://github.com/marketplace?category=testing&query=&type=&verification=',
    'Testing'
  ],
  [
    'https://github.com/marketplace?category=utilities&query=&type=&verification=',
    'Utilities'
  ],
  [ 'https://github.com/marketplace/category/free', 'Free' ],
  [
    'https://github.com/marketplace/category/free-trials',
    'Free Trials'
  ],
  [
    'https://github.com/marketplace/category/github-enterprise',
    'GitHub Enterprise'
  ],
  [ 'https://github.com/marketplace/category/paid', 'Paid' ],
  [
    'https://github.com/marketplace?category=&query=&type=&verification=verified_creator',
    'Verified Creator'
  ],
  [
    'https://github.com/marketplace?category=&query=&type=apps&verification=',
    'Apps'
  ],
  [
    'https://github.com/marketplace?category=&query=&type=actions&verification=',
    'Actions'
  ],
  [
    'https://github.com/marketplace?category=api-management&query=&type=&verification=',
    'API management'
  ],
  [
    'https://github.com/marketplace?category=chat&query=&type=&verification=',
    'Chat'
  ],
  [
    'https://github.com/marketplace?category=code-quality&query=&type=&verification=',
    'Code quality'
  ],
  [
    'https://github.com/marketplace?category=code-review&query=&type=&verification=',
    'Code review'
  ],
  [
    'https://github.com/marketplace?category=continuous-integration&query=&type=&verification=',
    'Continuous integration'
  ],
  [
    'https://github.com/marketplace?category=dependency-management&query=&type=&verification=',
    'Dependency management'
  ],
  [
    'https://github.com/marketplace?category=deployment&query=&type=&verification=',
    'Deployment'
  ],
  [
    'https://github.com/marketplace?category=ides&query=&type=&verification=',
    'IDEs'
  ],
  [
    'https://github.com/marketplace?category=learning&query=&type=&verification=',
    'Learning'
  ],
  [
    'https://github.com/marketplace?category=localization&query=&type=&verification=',
    'Localization'
  ],
  [
    'https://github.com/marketplace?category=mobile&query=&type=&verification=',
    'Mobile'
  ],
  [
    'https://github.com/marketplace?category=monitoring&query=&type=&verification=',
    'Monitoring'
  ],
  [
    'https://github.com/marketplace?category=project-management&query=&type=&verification=',
    'Project management'
  ],
  [
    'https://github.com/marketplace?category=publishing&query=&type=&verification=',
    'Publishing'
  ],
  [
    'https://github.com/marketplace?category=recently-added&query=&type=&verification=',
    'Recently added'
  ],
  [
    'https://github.com/marketplace?category=security&query=&type=&verification=',
    'Security'
  ],
  [
    'https://github.com/marketplace?category=support&query=&type=&verification=',
    'Support'
  ],
  [
    'https://github.com/marketplace?category=testing&query=&type=&verification=',
    'Testing'
  ],
  [
    'https://github.com/marketplace?category=utilities&query=&type=&verification=',
    'Utilities'
  ],
  [ 'https://github.com/marketplace/category/free', 'Free' ],
  [
    'https://github.com/marketplace/category/free-trials',
    'Free Trials'
  ],
  [
    'https://github.com/marketplace/category/github-enterprise',
    'GitHub Enterprise'
  ],
  [ 'https://github.com/marketplace/category/paid', 'Paid' ],
  [
    'https://github.com/marketplace?category=&query=&type=&verification=verified_creator',
    'Verified Creator'
  ],
  [
    'https://github.com/marketplace/circleci',
    'CircleCI By circleci Automatically build, test, and deploy your project in minutes Recommended'
  ],
  [
    'https://github.com/marketplace/codefactor',
    'CodeFactor By codefactor-io Automated code review for GitHub Recommended'
  ],
  [
    'https://github.com/marketplace/buildpulse',
    'BuildPulse By Workshop64 Automatically detect, track, and rank flaky tests so you can regain trust in your test suite 44 installs'
  ],
  [
    'https://github.com/marketplace/cloudback',
    'Cloudback Backup By cloudback Backups your GitHub repositories, fast and secure 298 installs'
  ],
  [
    'https://github.com/marketplace/lzu-covid-health-report',
    'LZU COVID Health Report By Hollow-Software 兰州大学疫情期间自动定时健康打卡工作流 LZU Auto COVID Health Report Using GitHub Actions 406 installs'
  ],
  [
    'https://github.com/marketplace/gitprotect-io',
    'GitProtect.io Backup By xoperosoftware Fully manageable, most professional repository and metadata backup and recovery 388 installs'
  ],
  ... 57 more items
]