mirror/caddy
1
0
Fork 0
mirror of https://github.com/caddyserver/caddy.git synced 2026-05-19 14:08:31 -04:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: Record TLS ClientHellos by hash of key of structured data

Browse Source 
Also improve handling of disabled metrics, and record TLS ClientHello
in association with User-Agent
This commit is contained in:
Matthew Holt
2018-05-07 16:09:39 -06:00
parent 518edd3cd4
commit 078770a5a6
7 changed files with 211 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
caddyhttp/httpserver/mitm.go
View File

@@ -25,6 +25,7 @@ import (
"strings"
"sync"
"github.com/mholt/caddy/caddytls"
"github.com/mholt/caddy/telemetry"
)
@@ -65,6 +66,9 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ua := r.Header.Get("User-Agent")
// report this request's UA in connection with this ClientHello
go telemetry.AppendUnique("tls_client_hello_ua:"+info.Key(), ua)
var checked, mitm bool
if r.Header.Get("X-BlueCoat-Via") != "" || // Blue Coat (masks User-Agent header to generic values)
r.Header.Get("X-FCCKV2") != "" || // Fortinet
@@ -207,6 +211,11 @@ func (c *clientHelloConn) Read(b []byte) (n int, err error) {
c.listener.helloInfos[c.Conn.RemoteAddr().String()] = rawParsed
c.listener.helloInfosMu.Unlock()
// report this ClientHello to telemetry
chKey := rawParsed.Key()
go telemetry.SetNested("tls_client_hello", chKey, rawParsed)
go telemetry.AppendUnique("tls_client_hello_count", chKey)
c.readHello = true
return
}
@@ -227,6 +236,7 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
if len(data) < 42 {
return
}
info.Version = uint16(data[4])<<8 | uint16(data[5])
sessionIDLen := int(data[38])
if sessionIDLen > 32 || len(data) < 39+sessionIDLen {
return
@@ -243,9 +253,9 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
}
numCipherSuites := cipherSuiteLen / 2
// read in the cipher suites
info.cipherSuites = make([]uint16, numCipherSuites)
info.CipherSuites = make([]uint16, numCipherSuites)
for i := 0; i < numCipherSuites; i++ {
info.cipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i])
info.CipherSuites[i] = uint16(data[2+2*i])<<8 | uint16(data[3+2*i])
}
data = data[2+cipherSuiteLen:]
if len(data) < 1 {
@@ -256,7 +266,7 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
if len(data) < 1+compressionMethodsLen {
return
}
info.compressionMethods = data[1 : 1+compressionMethodsLen]
info.CompressionMethods = data[1 : 1+compressionMethodsLen]
data = data[1+compressionMethodsLen:]
@@ -284,7 +294,7 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
}
// record that the client advertised support for this extension
info.extensions = append(info.extensions, extension)
info.Extensions = append(info.Extensions, extension)
switch extension {
case extensionSupportedCurves:
@@ -297,10 +307,10 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
return
}
numCurves := l / 2
info.curves = make([]tls.CurveID, numCurves)
info.Curves = make([]tls.CurveID, numCurves)
d := data[2:]
for i := 0; i < numCurves; i++ {
info.curves[i] = tls.CurveID(d[0])<<8 | tls.CurveID(d[1])
info.Curves[i] = tls.CurveID(d[0])<<8 | tls.CurveID(d[1])
d = d[2:]
}
case extensionSupportedPoints:
@@ -312,8 +322,8 @@ func parseRawClientHello(data []byte) (info rawHelloInfo) {
if length != l+1 {
return
}
info.points = make([]uint8, l)
copy(info.points, data[1:])
info.Points = make([]uint8, l)
copy(info.Points, data[1:])
}
data = data[length:]
@@ -364,18 +374,12 @@ func (l *tlsHelloListener) Accept() (net.Conn, error) {
// by Durumeric, Halderman, et. al. in
// "The Security Impact of HTTPS Interception":
// https://jhalderm.com/pub/papers/interception-ndss17.pdf
type rawHelloInfo struct {
cipherSuites []uint16
extensions []uint16
compressionMethods []byte
curves []tls.CurveID
points []uint8
}
type rawHelloInfo struct{ caddytls.ClientHelloInfo }
// advertisesHeartbeatSupport returns true if info indicates
// that the client supports the Heartbeat extension.
func (info rawHelloInfo) advertisesHeartbeatSupport() bool {
for _, ext := range info.extensions {
for _, ext := range info.Extensions {
if ext == extensionHeartbeat {
return true
}
@@ -398,31 +402,31 @@ func (info rawHelloInfo) looksLikeFirefox() bool {
// Note: Firefox 55+ doesn't appear to advertise 0xFF03 (65283, short headers). It used to be between 5 and 13.
// Note: Firefox on Fedora (or RedHat) doesn't include ECC suites because of patent liability.
requiredExtensionsOrder := []uint16{23, 65281, 10, 11, 35, 16, 5, 13}
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) {
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.Extensions, true) {
return false
}
// We check for both presence of curves and their ordering.
requiredCurves := []tls.CurveID{29, 23, 24, 25}
if len(info.curves) < len(requiredCurves) {
if len(info.Curves) < len(requiredCurves) {
return false
}
for i := range requiredCurves {
if info.curves[i] != requiredCurves[i] {
if info.Curves[i] != requiredCurves[i] {
return false
}
}
if len(info.curves) > len(requiredCurves) {
if len(info.Curves) > len(requiredCurves) {
// newer Firefox (55 Nightly?) may have additional curves at end of list
allowedCurves := []tls.CurveID{256, 257}
for i := range allowedCurves {
if info.curves[len(requiredCurves)+i] != allowedCurves[i] {
if info.Curves[len(requiredCurves)+i] != allowedCurves[i] {
return false
}
}
}
if hasGreaseCiphers(info.cipherSuites) {
if hasGreaseCiphers(info.CipherSuites) {
return false
}
@@ -449,7 +453,7 @@ func (info rawHelloInfo) looksLikeFirefox() bool {
tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa
}
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false)
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.CipherSuites, false)
}
// looksLikeChrome returns true if info looks like a handshake
@@ -490,20 +494,20 @@ func (info rawHelloInfo) looksLikeChrome() bool {
TLS_DHE_RSA_WITH_AES_128_CBC_SHA: {}, // 0x33
TLS_DHE_RSA_WITH_AES_256_CBC_SHA: {}, // 0x39
}
for _, ext := range info.cipherSuites {
for _, ext := range info.CipherSuites {
if _, ok := chromeCipherExclusions[ext]; ok {
return false
}
}
// Chrome does not include curve 25 (CurveP521) (as of Chrome 56, Feb. 2017).
for _, curve := range info.curves {
for _, curve := range info.Curves {
if curve == 25 {
return false
}
}
if !hasGreaseCiphers(info.cipherSuites) {
if !hasGreaseCiphers(info.CipherSuites) {
return false
}
@@ -521,19 +525,19 @@ func (info rawHelloInfo) looksLikeEdge() bool {
// More specifically, the OCSP status request extension appears
// *directly* before the other two extensions, which occur in that
// order. (I contacted the authors for clarification and verified it.)
for i, ext := range info.extensions {
for i, ext := range info.Extensions {
if ext == extensionOCSPStatusRequest {
if len(info.extensions) <= i+2 {
if len(info.Extensions) <= i+2 {
return false
}
if info.extensions[i+1] != extensionSupportedCurves ||
info.extensions[i+2] != extensionSupportedPoints {
if info.Extensions[i+1] != extensionSupportedCurves ||
info.Extensions[i+2] != extensionSupportedPoints {
return false
}
}
}
for _, cs := range info.cipherSuites {
for _, cs := range info.CipherSuites {
// As of Feb. 2017, Edge does not have 0xff, but Avast adds it
if cs == scsvRenegotiation {
return false
@@ -544,7 +548,7 @@ func (info rawHelloInfo) looksLikeEdge() bool {
}
}
if hasGreaseCiphers(info.cipherSuites) {
if hasGreaseCiphers(info.CipherSuites) {
return false
}
@@ -570,23 +574,23 @@ func (info rawHelloInfo) looksLikeSafari() bool {
// We check for the presence and order of the extensions.
requiredExtensionsOrder := []uint16{10, 11, 13, 13172, 16, 5, 18, 23}
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) {
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.Extensions, true) {
// Safari on iOS 11 (beta) uses different set/ordering of extensions
requiredExtensionsOrderiOS11 := []uint16{65281, 0, 23, 13, 5, 13172, 18, 16, 11, 10}
if !assertPresenceAndOrdering(requiredExtensionsOrderiOS11, info.extensions, true) {
if !assertPresenceAndOrdering(requiredExtensionsOrderiOS11, info.Extensions, true) {
return false
}
} else {
// For these versions of Safari, expect TLS_EMPTY_RENEGOTIATION_INFO_SCSV first.
if len(info.cipherSuites) < 1 {
if len(info.CipherSuites) < 1 {
return false
}
if info.cipherSuites[0] != scsvRenegotiation {
if info.CipherSuites[0] != scsvRenegotiation {
return false
}
}
if hasGreaseCiphers(info.cipherSuites) {
if hasGreaseCiphers(info.CipherSuites) {
return false
}
@@ -611,19 +615,19 @@ func (info rawHelloInfo) looksLikeSafari() bool {
tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35
tls.TLS_RSA_WITH_AES_128_CBC_SHA, // 0x2f
}
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, true)
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.CipherSuites, true)
}
// looksLikeTor returns true if the info looks like a ClientHello from Tor browser
// (based on Firefox).
func (info rawHelloInfo) looksLikeTor() bool {
requiredExtensionsOrder := []uint16{10, 11, 16, 5, 13}
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.extensions, true) {
if !assertPresenceAndOrdering(requiredExtensionsOrder, info.Extensions, true) {
return false
}
// check for session tickets support; Tor doesn't support them to prevent tracking
for _, ext := range info.extensions {
for _, ext := range info.Extensions {
if ext == 35 {
return false
}
@@ -631,12 +635,12 @@ func (info rawHelloInfo) looksLikeTor() bool {
// We check for both presence of curves and their ordering, including
// an optional curve at the beginning (for Tor based on Firefox 52)
infoCurves := info.curves
if len(info.curves) == 4 {
if info.curves[0] != 29 {
infoCurves := info.Curves
if len(info.Curves) == 4 {
if info.Curves[0] != 29 {
return false
}
infoCurves = info.curves[1:]
infoCurves = info.Curves[1:]
}
requiredCurves := []tls.CurveID{23, 24, 25}
if len(infoCurves) < len(requiredCurves) {
@@ -648,7 +652,7 @@ func (info rawHelloInfo) looksLikeTor() bool {
}
}
if hasGreaseCiphers(info.cipherSuites) {
if hasGreaseCiphers(info.CipherSuites) {
return false
}
@@ -675,7 +679,7 @@ func (info rawHelloInfo) looksLikeTor() bool {
tls.TLS_RSA_WITH_AES_256_CBC_SHA, // 0x35
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, // 0xa
}
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.cipherSuites, false)
return assertPresenceAndOrdering(expectedCipherSuiteOrder, info.CipherSuites, false)
}
// assertPresenceAndOrdering will return true if candidateList contains

6
caddyhttp/httpserver/plugin.go
View File

@@ -67,6 +67,12 @@ func init() {
caddy.RegisterParsingCallback(serverType, "root", hideCaddyfile)
caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
// disable the caddytls package reporting ClientHellos
// to telemetry, since our MITM detector does this but
// with more information than the standard lib provides
// (as of May 2018)
caddytls.ClientHelloTelemetry = false
}
// hideCaddyfile hides the source/origin Caddyfile if it is within the

8
caddyhttp/httpserver/server.go
View File

@@ -349,8 +349,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}()
// TODO: Somehow report UA string in conjunction with TLS handshake, if any (and just once per connection)
go telemetry.AppendUnique("http_user_agent", r.Header.Get("User-Agent"))
// record the User-Agent string (with a cap on its length to mitigate attacks)
ua := r.Header.Get("User-Agent")
if len(ua) > 512 {
ua = ua[:512]
}
go telemetry.AppendUnique("http_user_agent", ua)
go telemetry.Increment("http_request_count")
// copy the original, unchanged URL into the context