Source: maintenance/findBrokenLinks.js

  1. /**
  2.  * @file This file checks all links on the WOT-terms site and reports broken links. It also creates a GitHub issue with the broken links.
  3.  * The script should be run from the root of the WOT-terms repository.
  4.  * Environment: NodeJS
  5.  * Usage: 
  6.  * $ node findBrokenLinks.js
  7.  * @author Kor Dwarshuis
  8.  * @version 1.0.0
  9.  * @since 2023-09-04
  10.  * @see https://www.npmjs.com/package/broken-link-checker
  11.  * @see https://github.com/stevenvachon/broken-link-checker
  12.  */
  14. require('dotenv').config();
  15. const { Octokit } = require('@octokit/core');
  16. const fs = require('fs');
  17. const { SiteChecker } = require('broken-link-checker');
  18. const { URL } = require('url');
  19. const path = require('path');
  21. /**********/
  22. /* CONFIG */
  24. const siteUrl = 'https://weboftrust.github.io/WOT-terms';
  25. const baseUrl = 'https://weboftrust.github.io';
  27. const outputDirectory = path.join(__dirname, '../logs');
  28. const outputFileName = 'brokenLinks.md';
  29. // const excludedSubdirectories = ['/WOT-terms/slack/'];
  30. const githubToken = process.env.GITHUB_ISSUE_AUTH_TOKEN;
  32. /* END CONFIG */
  33. /**************/
  35. const outputFilePath = path.join(outputDirectory, outputFileName);
  36. let brokenLinks = {};
  37. let fileContent = '';
  39. console.log('Start Link checking...');
  41. const siteChecker = new SiteChecker({
  42.     excludeExternalLinks: true,
  43.     maxSocketsPerHost: 10
  44. }, {
  45.     link: (result) => {
  46.         // Log every URL that is checked
  47.         console.log(`Checking link: ${result.url.resolved}`);
  49.         // Additionally, log if a link is broken
  50.         if (result.broken) {
  52.             // brokenLinks.push({
  53.             //     url: result.url.resolved,
  54.             //     brokenReason: result.brokenReason
  55.             // });
  57.             const urlObj = new URL(result.url.original, baseUrl);
  58.             const baseObj = new URL(result.base.original, baseUrl);
  60.             const href = urlObj.href;
  61.             if (!brokenLinks[href]) {
  62.                 brokenLinks[href] = [];
  63.             }
  64.             if (!brokenLinks[href].includes(baseObj.href)) {
  65.                 brokenLinks[href].push(baseObj.href);
  66.             }
  67.             console.log(`Broken link found: ${result.url.resolved} (${result.brokenReason}). Found on page: ${baseObj.href}`);
  68.         }
  70.     },
  71.     end: () => {
  72.         console.log("Finished checking site.");
  73.         console.log('Checking done! Writing to file...');
  75.         // Get ISO8601 timestamp
  76.         const getISO8601Timestamp = () => {
  77.             const now = new Date();
  78.             return now.toISOString();
  79.         };
  81.         const timestamp = getISO8601Timestamp();
  82.         const numberOfBrokenLinks = Object.keys(brokenLinks).length;
  83.         console.log('numberOfBrokenLinks: ', numberOfBrokenLinks);
  85.         // Format the output for the Markdown file
  86.         fileContent = `# Broken Links Report\n\nCreated: ${timestamp}\n\n`;
  87.         fileContent += `Total Broken Links Found: ${numberOfBrokenLinks}\n\n`;
  89.         let counter = 1; // Initialize counter variable outside the loop
  91.         for (const [brokenLink, foundOnPages] of Object.entries(brokenLinks)) {
  92.             let markdownBrokenLink = `[${brokenLink}](${brokenLink})`;
  93.             let pagesMarkdown = foundOnPages.map(page => `- [${page}](${page})`).join('\n');
  94.             pagesMarkdown += '\n\n';
  95.             fileContent += `## Broken Link #${counter}:\n${markdownBrokenLink}\n\nFound on Pages:\n\n${pagesMarkdown}\n`;
  96.             counter++; // Increment counter for the next broken link
  97.         }
  99.         // Check if directory exists, if not then create it
  100.         if (!fs.existsSync(outputDirectory)) {
  101.             fs.mkdirSync(outputDirectory, { recursive: true });
  102.         }
  104.         fs.writeFile(outputFilePath, fileContent, async (err) => {
  105.             if (err) {
  106.                 console.error('Error writing to file:', err);
  107.             } else {
  108.                 console.log(`Broken links and count written to ${outputFilePath}`);
  109.             }
  110.         });
  112.         console.log('Creating GitHub issue...');
  114.         // TODO: Create GitHub should not be inside the file write callback
  115.         // Create GitHub issue using Octokit
  116.         const issueData = {
  117.             title: 'Broken Links Report',
  118.             body: "Created: " + timestamp + "\n\n" + "Number of broken internal links: " + numberOfBrokenLinks + "\n\n" + "<a href='https://github.com/WebOfTrust/WOT-terms/blob/main/logs/brokenLinks.md'>See full list of broken internal links</a>.",
  119.         };
  121.         const octokit = new Octokit({
  122.             auth: githubToken
  123.         });
  125.         octokit.request('POST /repos/WebOfTrust/WOT-terms/issues', {
  126.             owner: 'WebOfTrust',
  127.             repo: 'WOT-terms',
  128.             title: issueData.title,
  129.             body: issueData.body,
  130.             // labels: [
  131.             //     'bug'
  132.             // ],
  133.             headers: {
  134.                 'X-GitHub-Api-Version': '2022-11-28'
  135.             }
  136.         });
  138.         console.log('GitHub issue created.');
  140.     }
  141. });
  143. siteChecker.enqueue(siteUrl);