How to intercept and modify the js Request?

Question

How to intercept and modify the js Request?

tofmm opened this issue 5 months ago · comments

What versions are you running?

$ go list -m github.com/chromedp/chromedp
github.com/chromedp/chromedp v0.9.3
$ google-chrome --version
Google Chrome 120.0.6099.109 
$ go version
go version go1.21.4 linux/amd64

What did you do? Include clear steps.

package main

import (
	"context"
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"net/http/httptest"
	"time"

	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/fetch"
	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
)

type networkEvents struct {
	requestWillBeSent         *network.EventRequestWillBeSent
	responseReceived          *network.EventResponseReceived
	responseReceivedExtraInfo *network.EventResponseReceivedExtraInfo
}

func main() {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("request:", r.URL)
		if r.URL.Path == "/index" {
			fmt.Fprintf(w, `
<html>
<body>
<h1>index</h1>
<script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
</body>
</html>`)
		}
	}))
	defer s.Close()

	opts := append(chromedp.DefaultExecAllocatorOptions[:],
		chromedp.DisableGPU,
		chromedp.NoDefaultBrowserCheck,
		chromedp.Flag("headless", true),
	)
	ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
	defer cancel()

	ctx, cancel = chromedp.NewContext(ctx) //chromedp.WithDebugf(log.Printf),

	defer cancel()

	eventsMap := make(map[network.RequestID]networkEvents)

	chromedp.ListenTarget(ctx, func(ev interface{}) {
		switch ev := ev.(type) {
		case *fetch.EventRequestPaused:
			go func() {
				c := chromedp.FromContext(ctx)
				newCtx := cdp.WithExecutor(ctx, c.Target)
				// how to get https://cdn.bootcdn.net/ajax/libs/jquery/3.7.1/jquery.min.js body and modify
				body, err := fetch.GetResponseBody(ev.RequestID).Do(newCtx)
				if err != nil {
					log.Println(err)
				} else {
					modified := append(body, []byte("\n window['test'] = 'test'; ")...)
					status := 200
					modifiedLen := len(string(modified))
					headers := make([]*fetch.HeaderEntry, 0)
					headers = append(headers, &fetch.HeaderEntry{Name: "Connection", Value: "closed"})
					headers = append(headers, &fetch.HeaderEntry{Name: "Content-Length", Value: string(modifiedLen)})
					headers = append(headers, &fetch.HeaderEntry{Name: "Content-Type", Value: "text/javascript"})
					fetch.FulfillRequest(ev.RequestID, int64(status)).
						WithBody(base64.StdEncoding.EncodeToString([]byte(modified))).
						WithResponseHeaders(headers).
						WithResponsePhrase("OK").
						Do(ctx)
				}

			}()
		case *network.EventResponseReceived:
			e, ok := eventsMap[ev.RequestID]
			if !ok {
				e = networkEvents{responseReceived: ev}
			} else {
				e.responseReceived = ev
			}
			eventsMap[ev.RequestID] = e

		case *network.EventLoadingFinished:
			if _, ok := eventsMap[ev.RequestID]; !ok {
				break
			}

		}
	})

	var patterns []*fetch.RequestPattern
	patterns = append(patterns, &fetch.RequestPattern{
		URLPattern: "*jquery.min.js*",
	})
	var test string
	if err := chromedp.Run(ctx,
		fetch.Enable().WithPatterns(patterns),
		network.Enable(),
		chromedp.Navigate(s.URL+"/index"),
		chromedp.Evaluate(`window.test;`, &test),
	); err != nil {
		log.Println(err)
	}

	fmt.Println(test)

	time.Sleep(2 * time.Second)
}

What did you expect to see?

test

What did you see instead?

fetch.GetResponseBody not work

Can only get response body on HeadersReceived pattern matched requests. (-32000)