Figma Puppeteer Scraping
URLs
Meow
const cli = meow ( `
Usage
$ twitch <cmd>
Commands
list live
list vods
list clips
Flags
--pretty Print the list in a nicer format, when available.
Examples
$ twitch list live --pretty
` ,
// @ts -ignore
{
flags : {
url : {
type : "string" ,
alias : 'u' ,
default : DEFAULT_URL ,
isMultiple : false ,
isRequired : true
} ,
remoteDebug : {
type : "boolean" ,
alias : "r" ,
isRequired : false
}
}
} )
Unused JS
// const scriptResponse = await client.send('Runtime.runScript', {
// scriptId: 'figma-scraper',
// awaitPromise: true,
// })
// minimize
// await client.send('Browser.setWindowBounds', {windowId, bounds: {windowState: 'minimized'}});
// intercept(page, urrns, transform)
// browser.on('targetcreated', async (target) => {
// const page = await target.page()
// console.log('TAGET CREATED, INTERCEPT?', page)
// })
// in page eval
const scriptEl = document . createElement ( 'script' )
scriptEl . id = 'figma-scraper'
scriptEl . innerHTML = `console.log('figma',figma)`
document . body . appendChild ( scriptEl )
const scripts = body . querySelectorAll ( 'script' )
const s = document . querySelector ( '#figma-scraper' )
console . log ( scriptEl . innerHTML , s ?. innerHTML )
// Puppeteer browser create instead of connect
// headless:false,
// defaultViewport:null,
// devtools: true,
// args: ['--window-size=1920,1170','--window-position=0,0']
// INTERCEPTION
const requestCache = new Map ( ) ;
const urlPatterns = [
'*'
]
function transform ( source ) {
return prettier . format ( source , { parser :'babel' } ) ;
}
async function intercept ( page , patterns , transform ) {
const client = await page . target ( ) . createCDPSession ( ) ;
await client . send ( 'Network.enable' ) ;
await client . send ( 'Network.setRequestInterception' , {
patterns : patterns . map ( pattern => ( {
urlPattern : pattern , resourceType : 'Script' , interceptionStage : 'HeadersReceived'
} ) )
} ) ;
client . on ( 'Network.requestIntercepted' , async ( { interceptionId, request, responseHeaders, resourceType } ) => {
console . log ( `Intercepted ${ request . url } {interception id: ${ interceptionId } }` ) ;
const response = await client . send ( 'Network.getResponseBodyForInterception' , { interceptionId } ) ;
const contentTypeHeader = Object . keys ( responseHeaders ) . find ( k => k . toLowerCase ( ) === 'content-type' ) ;
// @ts -ignore
let newBody , contentType = responseHeaders [ contentTypeHeader ] ;
if ( requestCache . has ( response . body ) ) {
newBody = requestCache . get ( response . body ) ;
} else {
const bodyData = response . base64Encoded ? atob ( response . body ) : response . body ;
try {
if ( resourceType === 'Script' ) newBody = transform ( bodyData , { parser : 'babel' } ) ;
else newBody === bodyData
} catch ( e ) {
console . log ( `Failed to process ${ request . url } {interception id: ${ interceptionId } }: ${ e } ` ) ;
newBody = bodyData
}
requestCache . set ( response . body , newBody ) ;
}
const newHeaders = [
'Date: ' + ( new Date ( ) ) . toUTCString ( ) ,
'Connection: closed' ,
'Content-Length: ' + newBody . length ,
'Content-Type: ' + contentType
] ;
console . log ( `Continuing interception ${ interceptionId } ` )
client . send ( 'Network.continueInterceptedRequest' , {
interceptionId,
rawResponse : btoa ( 'HTTP/1.1 200 OK' + '\r\n' + newHeaders . join ( '\r\n' ) + '\r\n\r\n' + newBody )
} ) ;
} ) ;
}