From 6f290154229b1fa638dd5fdc1152f7e66e5e8036 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Mon, 31 Mar 2025 00:42:01 -0700 Subject: [PATCH] Set up a mostly-default project with BackstopJS Signed-off-by: Danila Fedorin --- backstop.js | 59 + backstop_data/engine_scripts/cookies.json | 14 + backstop_data/engine_scripts/imageStub.jpg | Bin 0 -> 2900 bytes .../playwright/clickAndHoverHelper.js | 43 + .../playwright/interceptImages.js | 31 + .../engine_scripts/playwright/loadCookies.js | 16 + .../engine_scripts/playwright/onBefore.js | 3 + .../engine_scripts/playwright/onReady.js | 6 + .../engine_scripts/playwright/overrideCSS.js | 27 + .../puppet/clickAndHoverHelper.js | 41 + .../engine_scripts/puppet/ignoreCSP.js | 65 + .../engine_scripts/puppet/interceptImages.js | 37 + .../engine_scripts/puppet/loadCookies.js | 33 + .../engine_scripts/puppet/onBefore.js | 3 + .../engine_scripts/puppet/onReady.js | 6 + .../engine_scripts/puppet/overrideCSS.js | 15 + chatgpt-fix-root-URLs.py | 74 + functions.bash | 3 + package-lock.json | 2613 +++++++++++++++++ package.json | 6 + required.txt | 4 + 21 files changed, 3099 insertions(+) create mode 100644 backstop.js create mode 100644 backstop_data/engine_scripts/cookies.json create mode 100644 backstop_data/engine_scripts/imageStub.jpg create mode 100644 backstop_data/engine_scripts/playwright/clickAndHoverHelper.js create mode 100644 backstop_data/engine_scripts/playwright/interceptImages.js create mode 100644 backstop_data/engine_scripts/playwright/loadCookies.js create mode 100644 backstop_data/engine_scripts/playwright/onBefore.js create mode 100644 backstop_data/engine_scripts/playwright/onReady.js create mode 100644 backstop_data/engine_scripts/playwright/overrideCSS.js create mode 100644 backstop_data/engine_scripts/puppet/clickAndHoverHelper.js create mode 100644 backstop_data/engine_scripts/puppet/ignoreCSP.js create mode 100644 backstop_data/engine_scripts/puppet/interceptImages.js create mode 100644 backstop_data/engine_scripts/puppet/loadCookies.js create mode 100644 backstop_data/engine_scripts/puppet/onBefore.js create mode 100644 backstop_data/engine_scripts/puppet/onReady.js create mode 100644 backstop_data/engine_scripts/puppet/overrideCSS.js create mode 100644 chatgpt-fix-root-URLs.py create mode 100644 functions.bash create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 required.txt diff --git a/backstop.js b/backstop.js new file mode 100644 index 0000000..f364a77 --- /dev/null +++ b/backstop.js @@ -0,0 +1,59 @@ +const fs = require('node:fs'); +const list = fs.readFileSync('./pages.json', 'utf8'); + +function scenarioForFile(file) { + return { + "label": file, + "cookiePath": "backstop_data/engine_scripts/cookies.json", + "url": file, + "referenceUrl": "", + "readyEvent": "", + "readySelector": "", + "delay": 0, + "hideSelectors": [], + "removeSelectors": [], + "hoverSelector": "", + "clickSelector": "", + "postInteractionWait": 0, + "selectors": [], + "selectorExpansion": true, + "expect": 0, + "misMatchThreshold" : 0.1, + "requireSameDimensions": true + }; +} + +module.exports = { + "id": "blog_regression", + "viewports": [ + { + "label": "phone", + "width": 320, + "height": 480 + }, + { + "label": "tablet", + "width": 1024, + "height": 768 + } + ], + "onBeforeScript": "puppet/onBefore.js", + "onReadyScript": "puppet/onReady.js", + "scenarios": list.map(scenarioForFile), + "paths": { + "bitmaps_reference": "backstop_data/bitmaps_reference", + "bitmaps_test": "backstop_data/bitmaps_test", + "engine_scripts": "backstop_data/engine_scripts", + "html_report": "backstop_data/html_report", + "ci_report": "backstop_data/ci_report" + }, + "report": ["browser"], + "engine": "puppeteer", + "engineOptions": { + "args": ["--no-sandbox"] + }, + "asyncCaptureLimit": 5, + "asyncCompareLimit": 50, + "debug": false, + "debugWindow": false +} diff --git a/backstop_data/engine_scripts/cookies.json b/backstop_data/engine_scripts/cookies.json new file mode 100644 index 0000000..4123a5d --- /dev/null +++ b/backstop_data/engine_scripts/cookies.json @@ -0,0 +1,14 @@ +[ + { + "domain": ".www.yourdomain.com", + "path": "/", + "name": "yourCookieName", + "value": "yourCookieValue", + "expirationDate": 1798790400, + "hostOnly": false, + "httpOnly": false, + "secure": false, + "session": false, + "sameSite": "Lax" + } +] diff --git a/backstop_data/engine_scripts/imageStub.jpg b/backstop_data/engine_scripts/imageStub.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e526b4bbbc74ef5125cb58f7c8849497ef4fa83 GIT binary patch literal 2900 zcmex=9G120;#n6owRLMnMK9K}Kdl#{WkcWEmJ4m_dk<0S4IEIarxlI2oD#A7QW+ zU|?WlWMbrCXJu!FC}3n_W?^Mx7vvC9G&C-r^5fY7?eLVKk@3AEqr{Jz%Yyo6b!Y!onr?BtIla@emFslL z-S8FrR5j%T&96$kfBO8-=%>8Z$In&<&yH_MnAi0?DAL$ST{2Qrt#*ly-l98|f2qq7|7NAhkHu#c>>Aju!aRdnziw1IC(W9m)HTZJYq`Z00B zv70fkX0JPPIp*?%QbowN@7(+#d8L*02a8`% z-z0AOec0^jFV+>ex(muBj!EWMhAq2v(YsaV+Re50XXgG>-O1N*GwhJWEziGit{Pik ze(BCA;rnfV!NXABNmouS*_v0fU7Rnurd?>NT= z#a*AhGJB`BymSAf(ra$HwE8~-+r_5&2jZVB(A#t>r|9IZ-`*M@(oMH7NqqV#Z0_lz zJG*&WMIXiIguHIfoP6rlgI}Ab^}M@$aN>lp&52Q`o*k;_xm`QysbY%z#y7P~l5U;Y zwdA4cVwa6s*=yxAe(^2{yxYBa$8LW`-yI#xyqtD|lG53gXw0`R+Tc7sMTeuJ$@yZ#|6}WahKD~)7X4vhU|?4GXnSq;-m~+WWj!z6Z#)jk zQsVbHZ>?^BB;|Is_L_Tqq{VW(hsP&p#OLUAx}$hpCdQ!mr-c%QQN)D%tp?1u3eUeodoBo^s>X@{1+f|2+C$%4*OC1lA~1IdzT?WYyi3oPVs= zX6m*@#?OqtOFYv3aB8u6g|U(0vZb89Tdu3^cdk6#vap}yWX2^O$K~Rn(oT}S^WB!a z?{a=fyz=Y&!)Ld%S|VrPhn4Qj#L}x4zw>-+$ntZt`^2oo?GqG}-`K`DWs`8Twncy6)Y2s&m(yD3iLXoBltV?Y||h zTm4Dts^~ml)s@pP%cZ6~_KtfQSR!x8`S_r~zO)UO)~zdi_hP=`iQShUSH>+%ef35u z#55=XlpV1r98rxght9kYsXb}COkuuJ#wV*YyQXY&N#EGxd(|&ESNXy=zl%qYAL%VS z@3!^ddG1rH7HKh0^fkMCgwj@T&1?NxylMNYs(SYi>V^u88@{YwJ*Rq?`k5lRFF)>u zubkPnn`0_($P|#}3@2>-OA!@8;n&kQ4yS%^^|CeCY<-pWUnkUd6@$WVYmJk2Hpzuc z^IX;iio1Uov{XH_R_%Q9mD%fGO#PSntAB3A&-qnrswS8I721E+KH&B7Go_IyrVHLM zib?!ZBi5w($!dQm_*R_~=V#S^rPD`^x^A5^OFmI^;gzpj<`c;; z^AomBiLT*%bjf8K&$FCGi(E=S-HQ9IoxA<{4YzA^B-M>?$TJ3}{=0Pk=R6lZa}VvSxV~0hd~y8Bl3A0k%H7yv2TFKY z%h^LqZZs*(y^yABtVP{#EGlKbHCzPIkMrUu~-H zaj4of$tIIiWmjL=6Zid|GdohgrmSwVS~U43if>M`+Vws)-&&ZXcsTRKCILUQ#kDGF zv2Dq|XYR}PH&3+5ev?&LWxwgZP{d_R-5`^F?7?c?yernvXvzyIMZRj8wX{ypc} zX5V5XZ~Oh3CAVS9?D%qK^{B!wpZ7eR5ey6rY>tnr*P2()wqvuJym&uz|GKzi()xR} z0$;YytjP4=xTfa1jce=OP4D)dG!U57z { + const hoverSelector = scenario.hoverSelectors || scenario.hoverSelector; + const clickSelector = scenario.clickSelectors || scenario.clickSelector; + const keyPressSelector = scenario.keyPressSelectors || scenario.keyPressSelector; + const scrollToSelector = scenario.scrollToSelector; + const postInteractionWait = scenario.postInteractionWait; // selector [str] | ms [int] + + if (keyPressSelector) { + for (const keyPressSelectorItem of [].concat(keyPressSelector)) { + await page.waitForSelector(keyPressSelectorItem.selector); + await page.type(keyPressSelectorItem.selector, keyPressSelectorItem.keyPress); + } + } + + if (hoverSelector) { + for (const hoverSelectorIndex of [].concat(hoverSelector)) { + await page.waitForSelector(hoverSelectorIndex); + await page.hover(hoverSelectorIndex); + } + } + + if (clickSelector) { + for (const clickSelectorIndex of [].concat(clickSelector)) { + await page.waitForSelector(clickSelectorIndex); + await page.click(clickSelectorIndex); + } + } + + if (postInteractionWait) { + if (parseInt(postInteractionWait) > 0) { + await page.waitForTimeout(postInteractionWait); + } else { + await page.waitForSelector(postInteractionWait); + } + } + + if (scrollToSelector) { + await page.waitForSelector(scrollToSelector); + await page.evaluate(scrollToSelector => { + document.querySelector(scrollToSelector).scrollIntoView(); + }, scrollToSelector); + } +}; diff --git a/backstop_data/engine_scripts/playwright/interceptImages.js b/backstop_data/engine_scripts/playwright/interceptImages.js new file mode 100644 index 0000000..4077e76 --- /dev/null +++ b/backstop_data/engine_scripts/playwright/interceptImages.js @@ -0,0 +1,31 @@ +/** + * INTERCEPT IMAGES + * Listen to all requests. If a request matches IMAGE_URL_RE + * then stub the image with data from IMAGE_STUB_URL + * + * Use this in an onBefore script E.G. + ``` + module.exports = async function(page, scenario) { + require('./interceptImages')(page, scenario); + } + ``` + * + */ + +const fs = require('fs'); +const path = require('path'); + +const IMAGE_URL_RE = /\.gif|\.jpg|\.png/i; +const IMAGE_STUB_URL = path.resolve(__dirname, '../../imageStub.jpg'); +const IMAGE_DATA_BUFFER = fs.readFileSync(IMAGE_STUB_URL); +const HEADERS_STUB = {}; + +module.exports = async function (page, scenario) { + page.route(IMAGE_URL_RE, route => { + route.fulfill({ + body: IMAGE_DATA_BUFFER, + headers: HEADERS_STUB, + status: 200 + }); + }); +}; diff --git a/backstop_data/engine_scripts/playwright/loadCookies.js b/backstop_data/engine_scripts/playwright/loadCookies.js new file mode 100644 index 0000000..6a9044e --- /dev/null +++ b/backstop_data/engine_scripts/playwright/loadCookies.js @@ -0,0 +1,16 @@ +const fs = require('fs'); + +module.exports = async (browserContext, scenario) => { + let cookies = []; + const cookiePath = scenario.cookiePath; + + // Read Cookies from File, if exists + if (fs.existsSync(cookiePath)) { + cookies = JSON.parse(fs.readFileSync(cookiePath)); + } + + // Add cookies to browser + browserContext.addCookies(cookies); + + console.log('Cookie state restored with:', JSON.stringify(cookies, null, 2)); +}; diff --git a/backstop_data/engine_scripts/playwright/onBefore.js b/backstop_data/engine_scripts/playwright/onBefore.js new file mode 100644 index 0000000..f163c2d --- /dev/null +++ b/backstop_data/engine_scripts/playwright/onBefore.js @@ -0,0 +1,3 @@ +module.exports = async (page, scenario, viewport, isReference, browserContext) => { + await require('./loadCookies')(browserContext, scenario); +}; diff --git a/backstop_data/engine_scripts/playwright/onReady.js b/backstop_data/engine_scripts/playwright/onReady.js new file mode 100644 index 0000000..a944d91 --- /dev/null +++ b/backstop_data/engine_scripts/playwright/onReady.js @@ -0,0 +1,6 @@ +module.exports = async (page, scenario, viewport, isReference, browserContext) => { + console.log('SCENARIO > ' + scenario.label); + await require('./clickAndHoverHelper')(page, scenario); + + // add more ready handlers here... +}; diff --git a/backstop_data/engine_scripts/playwright/overrideCSS.js b/backstop_data/engine_scripts/playwright/overrideCSS.js new file mode 100644 index 0000000..a61cbef --- /dev/null +++ b/backstop_data/engine_scripts/playwright/overrideCSS.js @@ -0,0 +1,27 @@ +/** + * OVERRIDE CSS + * Apply this CSS to the loaded page, as a way to override styles. + * + * Use this in an onReady script E.G. + ``` + module.exports = async function(page, scenario) { + await require('./overrideCSS')(page, scenario); + } + ``` + * + */ + +const BACKSTOP_TEST_CSS_OVERRIDE = ` + html { + background-image: none; + } +`; + +module.exports = async (page, scenario) => { + // inject arbitrary css to override styles + await page.addStyleTag({ + content: BACKSTOP_TEST_CSS_OVERRIDE + }); + + console.log('BACKSTOP_TEST_CSS_OVERRIDE injected for: ' + scenario.label); +}; diff --git a/backstop_data/engine_scripts/puppet/clickAndHoverHelper.js b/backstop_data/engine_scripts/puppet/clickAndHoverHelper.js new file mode 100644 index 0000000..703d3b8 --- /dev/null +++ b/backstop_data/engine_scripts/puppet/clickAndHoverHelper.js @@ -0,0 +1,41 @@ +module.exports = async (page, scenario) => { + const hoverSelector = scenario.hoverSelectors || scenario.hoverSelector; + const clickSelector = scenario.clickSelectors || scenario.clickSelector; + const keyPressSelector = scenario.keyPressSelectors || scenario.keyPressSelector; + const scrollToSelector = scenario.scrollToSelector; + const postInteractionWait = scenario.postInteractionWait; // selector [str] | ms [int] + + if (keyPressSelector) { + for (const keyPressSelectorItem of [].concat(keyPressSelector)) { + await page.waitForSelector(keyPressSelectorItem.selector); + await page.type(keyPressSelectorItem.selector, keyPressSelectorItem.keyPress); + } + } + + if (hoverSelector) { + for (const hoverSelectorIndex of [].concat(hoverSelector)) { + await page.waitForSelector(hoverSelectorIndex); + await page.hover(hoverSelectorIndex); + } + } + + if (clickSelector) { + for (const clickSelectorIndex of [].concat(clickSelector)) { + await page.waitForSelector(clickSelectorIndex); + await page.click(clickSelectorIndex); + } + } + + if (postInteractionWait) { + await new Promise(resolve => { + setTimeout(resolve, postInteractionWait); + }); + } + + if (scrollToSelector) { + await page.waitForSelector(scrollToSelector); + await page.evaluate(scrollToSelector => { + document.querySelector(scrollToSelector).scrollIntoView(); + }, scrollToSelector); + } +}; diff --git a/backstop_data/engine_scripts/puppet/ignoreCSP.js b/backstop_data/engine_scripts/puppet/ignoreCSP.js new file mode 100644 index 0000000..1dea285 --- /dev/null +++ b/backstop_data/engine_scripts/puppet/ignoreCSP.js @@ -0,0 +1,65 @@ +/** + * IGNORE CSP HEADERS + * Listen to all requests. If a request matches scenario.url + * then fetch the request again manually, strip out CSP headers + * and respond to the original request without CSP headers. + * Allows `ignoreHTTPSErrors: true` BUT... requires `debugWindow: true` + * + * see https://github.com/GoogleChrome/puppeteer/issues/1229#issuecomment-380133332 + * this is the workaround until Page.setBypassCSP lands... https://github.com/GoogleChrome/puppeteer/pull/2324 + * + * @param {REQUEST} request + * @return {VOID} + * + * Use this in an onBefore script E.G. + ``` + module.exports = async function(page, scenario) { + require('./removeCSP')(page, scenario); + } + ``` + * + */ + +const fetch = require('node-fetch'); +const https = require('https'); +const agent = new https.Agent({ + rejectUnauthorized: false +}); + +module.exports = async function (page, scenario) { + const intercept = async (request, targetUrl) => { + const requestUrl = request.url(); + + // FIND TARGET URL REQUEST + if (requestUrl === targetUrl) { + const cookiesList = await page.cookies(requestUrl); + const cookies = cookiesList.map(cookie => `${cookie.name}=${cookie.value}`).join('; '); + const headers = Object.assign(request.headers(), { cookie: cookies }); + const options = { + headers, + body: request.postData(), + method: request.method(), + follow: 20, + agent + }; + + const result = await fetch(requestUrl, options); + + const buffer = await result.buffer(); + const cleanedHeaders = result.headers._headers || {}; + cleanedHeaders['content-security-policy'] = ''; + await request.respond({ + body: buffer, + headers: cleanedHeaders, + status: result.status + }); + } else { + request.continue(); + } + }; + + await page.setRequestInterception(true); + page.on('request', req => { + intercept(req, scenario.url); + }); +}; diff --git a/backstop_data/engine_scripts/puppet/interceptImages.js b/backstop_data/engine_scripts/puppet/interceptImages.js new file mode 100644 index 0000000..2b02be9 --- /dev/null +++ b/backstop_data/engine_scripts/puppet/interceptImages.js @@ -0,0 +1,37 @@ +/** + * INTERCEPT IMAGES + * Listen to all requests. If a request matches IMAGE_URL_RE + * then stub the image with data from IMAGE_STUB_URL + * + * Use this in an onBefore script E.G. + ``` + module.exports = async function(page, scenario) { + require('./interceptImages')(page, scenario); + } + ``` + * + */ + +const fs = require('fs'); +const path = require('path'); + +const IMAGE_URL_RE = /\.gif|\.jpg|\.png/i; +const IMAGE_STUB_URL = path.resolve(__dirname, '../imageStub.jpg'); +const IMAGE_DATA_BUFFER = fs.readFileSync(IMAGE_STUB_URL); +const HEADERS_STUB = {}; + +module.exports = async function (page, scenario) { + const intercept = async (request, targetUrl) => { + if (IMAGE_URL_RE.test(request.url())) { + await request.respond({ + body: IMAGE_DATA_BUFFER, + headers: HEADERS_STUB, + status: 200 + }); + } else { + request.continue(); + } + }; + await page.setRequestInterception(true); + page.on('request', intercept); +}; diff --git a/backstop_data/engine_scripts/puppet/loadCookies.js b/backstop_data/engine_scripts/puppet/loadCookies.js new file mode 100644 index 0000000..85b90cd --- /dev/null +++ b/backstop_data/engine_scripts/puppet/loadCookies.js @@ -0,0 +1,33 @@ +const fs = require('fs'); + +module.exports = async (page, scenario) => { + let cookies = []; + const cookiePath = scenario.cookiePath; + + // READ COOKIES FROM FILE IF EXISTS + if (fs.existsSync(cookiePath)) { + cookies = JSON.parse(fs.readFileSync(cookiePath)); + } + + // MUNGE COOKIE DOMAIN + cookies = cookies.map(cookie => { + if (cookie.domain.startsWith('http://') || cookie.domain.startsWith('https://')) { + cookie.url = cookie.domain; + } else { + cookie.url = 'https://' + cookie.domain; + } + delete cookie.domain; + return cookie; + }); + + // SET COOKIES + const setCookies = async () => { + return Promise.all( + cookies.map(async (cookie) => { + await page.setCookie(cookie); + }) + ); + }; + await setCookies(); + console.log('Cookie state restored with:', JSON.stringify(cookies, null, 2)); +}; diff --git a/backstop_data/engine_scripts/puppet/onBefore.js b/backstop_data/engine_scripts/puppet/onBefore.js new file mode 100644 index 0000000..a1c374c --- /dev/null +++ b/backstop_data/engine_scripts/puppet/onBefore.js @@ -0,0 +1,3 @@ +module.exports = async (page, scenario, vp) => { + await require('./loadCookies')(page, scenario); +}; diff --git a/backstop_data/engine_scripts/puppet/onReady.js b/backstop_data/engine_scripts/puppet/onReady.js new file mode 100644 index 0000000..517c0e4 --- /dev/null +++ b/backstop_data/engine_scripts/puppet/onReady.js @@ -0,0 +1,6 @@ +module.exports = async (page, scenario, vp) => { + console.log('SCENARIO > ' + scenario.label); + await require('./clickAndHoverHelper')(page, scenario); + + // add more ready handlers here... +}; diff --git a/backstop_data/engine_scripts/puppet/overrideCSS.js b/backstop_data/engine_scripts/puppet/overrideCSS.js new file mode 100644 index 0000000..d568205 --- /dev/null +++ b/backstop_data/engine_scripts/puppet/overrideCSS.js @@ -0,0 +1,15 @@ +const BACKSTOP_TEST_CSS_OVERRIDE = 'html {background-image: none;}'; + +module.exports = async (page, scenario) => { + // inject arbitrary css to override styles + await page.evaluate(`window._styleData = '${BACKSTOP_TEST_CSS_OVERRIDE}'`); + await page.evaluate(() => { + const style = document.createElement('style'); + style.type = 'text/css'; + const styleNode = document.createTextNode(window._styleData); + style.appendChild(styleNode); + document.head.appendChild(style); + }); + + console.log('BACKSTOP_TEST_CSS_OVERRIDE injected for: ' + scenario.label); +}; diff --git a/chatgpt-fix-root-URLs.py b/chatgpt-fix-root-URLs.py new file mode 100644 index 0000000..a95f3b2 --- /dev/null +++ b/chatgpt-fix-root-URLs.py @@ -0,0 +1,74 @@ +import os +from bs4 import BeautifulSoup +from urllib.parse import urlparse + +# Domains considered part of your site. +SITE_ROOT_URLS = ["https://danilafe.com/", "http://danilafe.com/"] +# The project root is the current working directory. +PROJECT_ROOT = os.getcwd() +HTML_EXTENSIONS = {".html", ".htm"} + +def convert_to_relative(url, base_filepath): + """ + Convert an absolute URL (including domain-relative URLs) to a relative path + appropriate for the HTML file at base_filepath. + """ + parsed = urlparse(url) + # If the URL is already relative, return it unchanged. + if not (url.startswith("/") or any(url.startswith(root) for root in SITE_ROOT_URLS)): + return url + + # If it's an absolute URL on danilafe.com, strip the domain. + for root_url in SITE_ROOT_URLS: + if url.startswith(root_url): + url = url[len(root_url):] + break + + # For domain-relative URLs (starting with "/"), remove the leading slash. + if url.startswith("/"): + url = url.lstrip("/") + + # Build the full filesystem path for the target resource. + target_path = os.path.normpath(os.path.join(PROJECT_ROOT, url)) + base_dir = os.path.dirname(base_filepath) + # Compute the relative path from the HTML file's directory to the target. + relative_path = os.path.relpath(target_path, start=base_dir) + return relative_path.replace(os.path.sep, "/") + +def process_html_file(filepath): + """Process a single HTML file to rewrite links, unwrap