Giters

refraction-networking / utls

Fork of the Go standard TLS library, providing low-level access to the ClientHello for mimicry purposes.

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

Weird observation regarding ClientId and Spec

BRUHItsABunny opened this issue · comments

commented 
// assume these vars are set somewhere properly
var config tls.config
var conn net.Conn

uConfig := &utls.Config{
	RootCAs:                     config.RootCAs,
	NextProtos:                  config.NextProtos,
	ServerName:                  config.ServerName,
	InsecureSkipVerify:          config.InsecureSkipVerify,
	DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled,
	KeyLogWriter:                os.Stdout,
}
uTLSConn := utls.UClient(conn, uConfig, utls.HelloChrome_120)

This works, I can dial Google and it gives no errors.

// assume these vars are set somewhere properly
var config tls.config
var conn net.Conn

uConfig := &utls.Config{
	RootCAs:                     config.RootCAs,
	NextProtos:                  config.NextProtos,
	ServerName:                  config.ServerName,
	InsecureSkipVerify:          config.InsecureSkipVerify,
	DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled,
	KeyLogWriter:                os.Stdout,
}

spec := utls.ClientHelloSpec{
	CipherSuites: []uint16{
		utls.GREASE_PLACEHOLDER,
		utls.TLS_AES_128_GCM_SHA256,
		utls.TLS_AES_256_GCM_SHA384,
		utls.TLS_CHACHA20_POLY1305_SHA256,
		utls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
		utls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
		utls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		utls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		utls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
		utls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		utls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
		utls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
		utls.TLS_RSA_WITH_AES_128_GCM_SHA256,
		utls.TLS_RSA_WITH_AES_256_GCM_SHA384,
		utls.TLS_RSA_WITH_AES_128_CBC_SHA,
		utls.TLS_RSA_WITH_AES_256_CBC_SHA,
	},
	CompressionMethods: []byte{
		0x00, // compressionNone
	},
	Extensions: utls.ShuffleChromeTLSExtensions([]utls.TLSExtension{
		&utls.UtlsGREASEExtension{},
		&utls.SNIExtension{},
		&utls.ExtendedMasterSecretExtension{},
		&utls.RenegotiationInfoExtension{Renegotiation: utls.RenegotiateOnceAsClient},
		&utls.SupportedCurvesExtension{Curves: []utls.CurveID{
			utls.GREASE_PLACEHOLDER,
			utls.X25519,
			utls.CurveP256,
			utls.CurveP384,
		}},
		&utls.SupportedPointsExtension{SupportedPoints: []byte{
			0x00, // pointFormatUncompressed
		}},
		&utls.SessionTicketExtension{},
		&utls.ALPNExtension{AlpnProtocols: []string{"h2", "http/1.1"}},
		&utls.StatusRequestExtension{},
		&utls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []utls.SignatureScheme{
			utls.ECDSAWithP256AndSHA256,
			utls.PSSWithSHA256,
			utls.PKCS1WithSHA256,
			utls.ECDSAWithP384AndSHA384,
			utls.PSSWithSHA384,
			utls.PKCS1WithSHA384,
			utls.PSSWithSHA512,
			utls.PKCS1WithSHA512,
		}},
		&utls.SCTExtension{},
		&utls.KeyShareExtension{KeyShares: []utls.KeyShare{
			{Group: utls.CurveID(utls.GREASE_PLACEHOLDER), Data: []byte{0}},
			{Group: utls.X25519},
		}},
		&utls.PSKKeyExchangeModesExtension{Modes: []uint8{
			utls.PskModeDHE,
		}},
		&utls.SupportedVersionsExtension{Versions: []uint16{
			utls.GREASE_PLACEHOLDER,
			utls.VersionTLS13,
			utls.VersionTLS12,
		}},
		&utls.UtlsCompressCertExtension{Algorithms: []utls.CertCompressionAlgo{
			utls.CertCompressionBrotli,
		}},
		&utls.ApplicationSettingsExtension{SupportedProtocols: []string{"h2"}},
		utls.BoringGREASEECH(),
		&utls.UtlsGREASEExtension{},
	}),
}

uTLSConn := utls.UClient(conn, uConfig, utls.HelloCustom)
if err := uTLSConn.ApplyPreset(&spec); err != nil {
	panic(fmt.Errorf("uTLSConn.ApplyPreset: %w", err))
}

This can't dial Google, it errors out with Get "https://www.google.com/": local error: tls: internal error
If I comment out utls.VersionTLS13, it will work though.
This spec is a copy paste of the spec backing utls.HelloChrome_120, and there TLS 1.3 functions as expected.

This code is a basic example of what I'm facing, I will post a repo with a piece of code that reproduces the error if needed.

Does anyone happen to know what's wrong? I'm not entirely sure what I'm missing to get this to work. The end goal will be to load custom specs from my database, similar to loading from JA3 except my database entries contain the missing information that's technically still part of the TLS hello but not fully represented in a JA3 full string. This technically does work already, when limiting to TLS 1.2 and below.

Thanks for your time

commented

Interesting. So you are basically saying you copy-pasted the preset of HelloChrome_120 from u_parrot.go and trying to use it as a custom ClientHelloSpec but with no luck.

While I have not yet encounter such problem in the past, here's a few thing you might want to double check:

  • Are you reusing your spec? Currently uTLS modifies the input spec, the solution is making a deep copy every time (#111).
  • Perhaps you are not setting config.PreferSkipResumptionOnNilExtension to true while not including PSK in your Config, when using a custom parrot.

    utls/u_conn.go

    Line 69 in d2768e4

    uconn.skipResumptionOnNilExtension = config.PreferSkipResumptionOnNilExtension || clientHelloID.Client != helloCustom

Plus, doing a pcap may efficiently assist you in figuring out in which step of the TLS handshake did the error occur. You may also want to trace back to which tls: internal error are you seeing. I think uTLS has a longer error message like tls: internal error: attempted to read record with pending application data instead of just tls: internal error.

commented

So you are basically saying you copy-pasted the preset of HelloChrome_120 from u_parrot.go and trying to use it as a custom ClientHelloSpec but with no luck.

That's correct, with the exception of commenting out TLS 1.3 in the supported TLS versions in the ClientSpec somehow not giving an error.

I can confirm that I am not re-using the ClientSpec as this is currently running inside a unit test that just makes a single client with a single request going out.

Good catch on that config.PreferSkipResumptionOnNilExtension this got my hopes up but despite adding it the issue remains the same.

I will try a pcap sometime today and post full code that reproduces this issue as well.

commented

Also I would suggest explicitly call uTLSConn.Handshake() after you call ApplyPreset() to see if it errors. So far I have tried a variety of combinations of possible utls.Config fields but was able to connect to www.google.com:443 anyways. If you can share a hard-coded Config you used to reproduce this error, either me or someone from the community might be able to help you.

commented
image It looks like the initial connection works but then the redirect to www.google.com is failing.

When using the ClientHelloId it seems to be able to handle that redirect:
image

I'm a bit rusty with wireshark, but I will try and dig deeper into this tomorrow

commented

Code that reproduces the issue can be found here:
https://github.com/BRUHItsABunny/gOkHttp-ja3spoof/blob/test/tls-1-3-issue/client_opt_test.go

I will work on a snippet that doesn't use oohttp to see if I can limit the possibilities of outside influences

commented

Excuse me for intrusion.

What caught my eye on these screenshots is that SNI is incorrect on redirected connection in the first case (it's google.com, although www.google.com is expected). DNS query is for www.google.com, so host is probably passed in correctly.
On the second screenshot second connection's SNI is correct.

My bet is that there's something wrong with SNI. I support @gaukas' suggestion about Handshake() and also suggest trying SetSNI() before Handshake() (at least that's the way I use custom ClientHello specs and it works).

commented

SNI is incorrect on redirected connection

Good catch. And I actually think it is an indication of an old client spec being reused -- or something else is happening which prevented the server name from updating.

commented
Code Reproduce 
package main

import (
	"fmt"
	"net"

	tls "github.com/refraction-networking/utls"
)

var spec tls.ClientHelloSpec = tls.ClientHelloSpec{
	CipherSuites: []uint16{
		tls.GREASE_PLACEHOLDER,
		tls.TLS_AES_128_GCM_SHA256,
		tls.TLS_AES_256_GCM_SHA384,
		tls.TLS_CHACHA20_POLY1305_SHA256,
		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
		tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
		tls.TLS_RSA_WITH_AES_128_CBC_SHA,
		tls.TLS_RSA_WITH_AES_256_CBC_SHA,
	},
	CompressionMethods: []byte{
		0x00, // compressionNone
	},
	Extensions: tls.ShuffleChromeTLSExtensions([]tls.TLSExtension{
		&tls.UtlsGREASEExtension{},
		&tls.SNIExtension{},
		&tls.ExtendedMasterSecretExtension{},
		&tls.RenegotiationInfoExtension{Renegotiation: tls.RenegotiateOnceAsClient},
		&tls.SupportedCurvesExtension{Curves: []tls.CurveID{
			tls.GREASE_PLACEHOLDER,
			tls.X25519,
			tls.CurveP256,
			tls.CurveP384,
		}},
		&tls.SupportedPointsExtension{SupportedPoints: []byte{
			0x00, // pointFormatUncompressed
		}},
		&tls.SessionTicketExtension{},
		&tls.ALPNExtension{AlpnProtocols: []string{"h2", "http/1.1"}},
		&tls.StatusRequestExtension{},
		&tls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []tls.SignatureScheme{
			tls.ECDSAWithP256AndSHA256,
			tls.PSSWithSHA256,
			tls.PKCS1WithSHA256,
			tls.ECDSAWithP384AndSHA384,
			tls.PSSWithSHA384,
			tls.PKCS1WithSHA384,
			tls.PSSWithSHA512,
			tls.PKCS1WithSHA512,
		}},
		&tls.SCTExtension{},
		&tls.KeyShareExtension{KeyShares: []tls.KeyShare{
			{Group: tls.CurveID(tls.GREASE_PLACEHOLDER), Data: []byte{0}},
			{Group: tls.X25519},
		}},
		&tls.PSKKeyExchangeModesExtension{Modes: []uint8{
			tls.PskModeDHE,
		}},
		&tls.SupportedVersionsExtension{Versions: []uint16{
			tls.GREASE_PLACEHOLDER,
			tls.VersionTLS13,
			tls.VersionTLS12,
		}},
		&tls.UtlsCompressCertExtension{Algorithms: []tls.CertCompressionAlgo{
			tls.CertCompressionBrotli,
		}},
		&tls.ApplicationSettingsExtension{SupportedProtocols: []string{"h2"}},
		tls.BoringGREASEECH(),
		&tls.UtlsGREASEExtension{},
	}),
}

func getSpec() *tls.ClientHelloSpec {
	return &tls.ClientHelloSpec{
		CipherSuites: []uint16{
			tls.GREASE_PLACEHOLDER,
			tls.TLS_AES_128_GCM_SHA256,
			tls.TLS_AES_256_GCM_SHA384,
			tls.TLS_CHACHA20_POLY1305_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
		},
		CompressionMethods: []byte{
			0x00, // compressionNone
		},
		Extensions: tls.ShuffleChromeTLSExtensions([]tls.TLSExtension{
			&tls.UtlsGREASEExtension{},
			&tls.SNIExtension{},
			&tls.ExtendedMasterSecretExtension{},
			&tls.RenegotiationInfoExtension{Renegotiation: tls.RenegotiateOnceAsClient},
			&tls.SupportedCurvesExtension{Curves: []tls.CurveID{
				tls.GREASE_PLACEHOLDER,
				tls.X25519,
				tls.CurveP256,
				tls.CurveP384,
			}},
			&tls.SupportedPointsExtension{SupportedPoints: []byte{
				0x00, // pointFormatUncompressed
			}},
			&tls.SessionTicketExtension{},
			&tls.ALPNExtension{AlpnProtocols: []string{"h2", "http/1.1"}},
			&tls.StatusRequestExtension{},
			&tls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []tls.SignatureScheme{
				tls.ECDSAWithP256AndSHA256,
				tls.PSSWithSHA256,
				tls.PKCS1WithSHA256,
				tls.ECDSAWithP384AndSHA384,
				tls.PSSWithSHA384,
				tls.PKCS1WithSHA384,
				tls.PSSWithSHA512,
				tls.PKCS1WithSHA512,
			}},
			&tls.SCTExtension{},
			&tls.KeyShareExtension{KeyShares: []tls.KeyShare{
				{Group: tls.CurveID(tls.GREASE_PLACEHOLDER), Data: []byte{0}},
				{Group: tls.X25519},
			}},
			&tls.PSKKeyExchangeModesExtension{Modes: []uint8{
				tls.PskModeDHE,
			}},
			&tls.SupportedVersionsExtension{Versions: []uint16{
				tls.GREASE_PLACEHOLDER,
				tls.VersionTLS13,
				tls.VersionTLS12,
			}},
			&tls.UtlsCompressCertExtension{Algorithms: []tls.CertCompressionAlgo{
				tls.CertCompressionBrotli,
			}},
			&tls.ApplicationSettingsExtension{SupportedProtocols: []string{"h2"}},
			tls.BoringGREASEECH(),
			&tls.UtlsGREASEExtension{},
		}),
	}
}

var config tls.Config = tls.Config{
	InsecureSkipVerify:          true,
	ServerName:                  "google.com",
	NextProtos:                  []string{"h2", "http/1.1"},
	DynamicRecordSizingDisabled: true,
}

func main() {
	tcpConn, err := net.Dial("tcp", "google.com:443")
	if err != nil {
		panic(err)
	}

	tlsConn := tls.UClient(tcpConn, &config, tls.HelloCustom)
	if err := tlsConn.ApplyPreset(&spec); err != nil {
		panic(fmt.Errorf("uTLSConn.ApplyPreset: %w", err))
	}

	if err := tlsConn.Handshake(); err != nil {
		panic(fmt.Errorf("uTLSConn.Handshake: %w", err))
	}

	tlsConn.Close()

	// then we handshake with www.google.com with the same setup
	config.ServerName = "www.google.com"
	tcpConn, err = net.Dial("tcp", "www.google.com:443")
	if err != nil {
		panic(err)
	}

	tlsConn = tls.UClient(tcpConn, &config, tls.HelloCustom)
	// if err := tlsConn.ApplyPreset(&spec); err != nil { // bad
	if err := tlsConn.ApplyPreset(getSpec()); err != nil { // good
		panic(fmt.Errorf("uTLSConn.ApplyPreset: %w", err))
	}

	if err := tlsConn.Handshake(); err != nil {
		panic(fmt.Errorf("uTLSConn.Handshake: %w", err))
	}
}

Reproduced error. And the error is from here:

utls/handshake_client_tls13.go

Lines 136 to 139 in d2768e4

if (hs.ecdheKey == nil && hs.kemKey == nil) || len(hs.hello.keyShares) < 1 { // [uTLS]
// keyshares "< 1" instead of "!= 1", as uTLS may send multiple
return c.sendAlert(alertInternalError)
}

No extra error message because it is not a specific error hardcoded inside uTLS, but rather a general TLS Alert alertInternalError (80).

Now I am almost certain that you reused the ClientHelloSpec. Let me know if otherwise @BRUHItsABunny.

commented

Thanks, it didn't look like a re-use at first glance but looking deeper into it it really is a re-use and I got PoC working that fixes that within my code base.
Thank you all very much for your help and patience.