Add metrics for tracking rendezvous method

Update tests for metrics

Add rendezvous_method to Prometheus metrics

Update broker spec docs with rendezvous method metrics

Bug fix

8 files changed, 355 insertions, 30 deletions

diff --git a/broker/amp.go b/broker/amp.go
index 4f047ec..b2def12 100644
--- a/broker/amp.go
+++ b/broker/amp.go
@@ -37,7 +37,7 @@ func ampClientOffers(i *IPC, w http.ResponseWriter, r *http.Request) {
 			Body:       encPollReq,
 			RemoteAddr: "",
 		}
-		err = i.ClientOffers(arg, &response)
+		err = i.ClientOffers(arg, &response, RendezvousAmpCache)
 	} else {
 		response, err = (&messages.ClientPollResponse{
 			Error: "cannot decode URL path",
diff --git a/broker/http.go b/broker/http.go
index 6057faa..ca9488f 100644
--- a/broker/http.go
+++ b/broker/http.go
@@ -172,7 +172,7 @@ func clientOffers(i *IPC, w http.ResponseWriter, r *http.Request) {
 	}
 
 	var response []byte
-	err = i.ClientOffers(arg, &response)
+	err = i.ClientOffers(arg, &response, RendezvousHttp)
 	if err != nil {
 		log.Println(err)
 		w.WriteHeader(http.StatusInternalServerError)
diff --git a/broker/ipc.go b/broker/ipc.go
index 132dba7..c61df1b 100644
--- a/broker/ipc.go
+++ b/broker/ipc.go
@@ -161,7 +161,7 @@ func sendClientResponse(resp *messages.ClientPollResponse, response *[]byte) err
 	}
 }
 
-func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte) error {
+func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte, rendezvousMethod RendezvousMethod) error {
 	startTime := time.Now()
 
 	req, err := messages.DecodeClientPollRequest(arg.Body)
@@ -195,12 +195,12 @@ func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte) error {
 		snowflake.offerChannel <- offer
 	} else {
 		i.ctx.metrics.lock.Lock()
-		i.ctx.metrics.clientDeniedCount++
-		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "denied"}).Inc()
+		i.ctx.metrics.clientDeniedCount[rendezvousMethod]++
+		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "denied", "rendezvous_method": string(rendezvousMethod)}).Inc()
 		if offer.natType == NATUnrestricted {
-			i.ctx.metrics.clientUnrestrictedDeniedCount++
+			i.ctx.metrics.clientUnrestrictedDeniedCount[rendezvousMethod]++
 		} else {
-			i.ctx.metrics.clientRestrictedDeniedCount++
+			i.ctx.metrics.clientRestrictedDeniedCount[rendezvousMethod]++
 		}
 		i.ctx.metrics.lock.Unlock()
 		resp := &messages.ClientPollResponse{Error: messages.StrNoProxies}
@@ -211,8 +211,8 @@ func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte) error {
 	select {
 	case answer := <-snowflake.answerChannel:
 		i.ctx.metrics.lock.Lock()
-		i.ctx.metrics.clientProxyMatchCount++
-		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "matched"}).Inc()
+		i.ctx.metrics.clientProxyMatchCount[rendezvousMethod]++
+		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "matched", "rendezvous_method": string(rendezvousMethod)}).Inc()
 		i.ctx.metrics.lock.Unlock()
 		resp := &messages.ClientPollResponse{Answer: answer}
 		err = sendClientResponse(resp, response)
diff --git a/broker/metrics.go b/broker/metrics.go
index 8788a96..5b0d3e5 100644
--- a/broker/metrics.go
+++ b/broker/metrics.go
@@ -36,6 +36,14 @@ type CountryStats struct {
 	counts map[string]int
 }
 
+type RendezvousMethod string
+
+const (
+	RendezvousHttp     RendezvousMethod = "http"
+	RendezvousAmpCache RendezvousMethod = "ampcache"
+	RendezvousSqs      RendezvousMethod = "sqs"
+)
+
 // Implements Observable
 type Metrics struct {
 	logger  *log.Logger
@@ -44,10 +52,10 @@ type Metrics struct {
 	countryStats                  CountryStats
 	clientRoundtripEstimate       time.Duration
 	proxyIdleCount                uint
-	clientDeniedCount             uint
-	clientRestrictedDeniedCount   uint
-	clientUnrestrictedDeniedCount uint
-	clientProxyMatchCount         uint
+	clientDeniedCount             map[RendezvousMethod]uint
+	clientRestrictedDeniedCount   map[RendezvousMethod]uint
+	clientUnrestrictedDeniedCount map[RendezvousMethod]uint
+	clientProxyMatchCount         map[RendezvousMethod]uint
 
 	proxyPollWithRelayURLExtension         uint
 	proxyPollWithoutRelayURLExtension      uint
@@ -152,6 +160,11 @@ func (m *Metrics) LoadGeoipDatabases(geoipDB string, geoip6DB string) error {
 func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
 	m := new(Metrics)
 
+	m.clientDeniedCount = make(map[RendezvousMethod]uint)
+	m.clientRestrictedDeniedCount = make(map[RendezvousMethod]uint)
+	m.clientUnrestrictedDeniedCount = make(map[RendezvousMethod]uint)
+	m.clientProxyMatchCount = make(map[RendezvousMethod]uint)
+
 	m.countryStats = CountryStats{
 		counts:          make(map[string]int),
 		proxies:         make(map[string]map[string]bool),
@@ -196,10 +209,19 @@ func (m *Metrics) printMetrics() {
 	m.logger.Println("snowflake-proxy-poll-with-relay-url-count", binCount(m.proxyPollWithRelayURLExtension))
 	m.logger.Println("snowflake-proxy-poll-without-relay-url-count", binCount(m.proxyPollWithoutRelayURLExtension))
 	m.logger.Println("snowflake-proxy-rejected-for-relay-url-count", binCount(m.proxyPollRejectedWithRelayURLExtension))
-	m.logger.Println("client-denied-count", binCount(m.clientDeniedCount))
-	m.logger.Println("client-restricted-denied-count", binCount(m.clientRestrictedDeniedCount))
-	m.logger.Println("client-unrestricted-denied-count", binCount(m.clientUnrestrictedDeniedCount))
-	m.logger.Println("client-snowflake-match-count", binCount(m.clientProxyMatchCount))
+
+	m.logger.Println("client-denied-count", binCount(sumMapValues(&m.clientDeniedCount)))
+	m.logger.Println("client-restricted-denied-count", binCount(sumMapValues(&m.clientRestrictedDeniedCount)))
+	m.logger.Println("client-unrestricted-denied-count", binCount(sumMapValues(&m.clientUnrestrictedDeniedCount)))
+	m.logger.Println("client-snowflake-match-count", binCount(sumMapValues(&m.clientProxyMatchCount)))
+
+	for _, rendezvousMethod := range [3]RendezvousMethod{RendezvousHttp, RendezvousAmpCache, RendezvousSqs} {
+		m.logger.Printf("client-%s-denied-count %d\n", rendezvousMethod, binCount(m.clientDeniedCount[rendezvousMethod]))
+		m.logger.Printf("client-%s-restricted-denied-count %d\n", rendezvousMethod, binCount(m.clientRestrictedDeniedCount[rendezvousMethod]))
+		m.logger.Printf("client-%s-unrestricted-denied-count %d\n", rendezvousMethod, binCount(m.clientUnrestrictedDeniedCount[rendezvousMethod]))
+		m.logger.Printf("client-snowflake-%s-match-count %d\n", rendezvousMethod, binCount(m.clientProxyMatchCount[rendezvousMethod]))
+	}
+
 	m.logger.Println("snowflake-ips-nat-restricted", len(m.countryStats.natRestricted))
 	m.logger.Println("snowflake-ips-nat-unrestricted", len(m.countryStats.natUnrestricted))
 	m.logger.Println("snowflake-ips-nat-unknown", len(m.countryStats.natUnknown))
@@ -209,13 +231,13 @@ func (m *Metrics) printMetrics() {
 // Restores all metrics to original values
 func (m *Metrics) zeroMetrics() {
 	m.proxyIdleCount = 0
-	m.clientDeniedCount = 0
-	m.clientRestrictedDeniedCount = 0
-	m.clientUnrestrictedDeniedCount = 0
+	m.clientDeniedCount = make(map[RendezvousMethod]uint)
+	m.clientRestrictedDeniedCount = make(map[RendezvousMethod]uint)
+	m.clientUnrestrictedDeniedCount = make(map[RendezvousMethod]uint)
 	m.proxyPollRejectedWithRelayURLExtension = 0
 	m.proxyPollWithRelayURLExtension = 0
 	m.proxyPollWithoutRelayURLExtension = 0
-	m.clientProxyMatchCount = 0
+	m.clientProxyMatchCount = make(map[RendezvousMethod]uint)
 	m.countryStats.counts = make(map[string]int)
 	for pType := range m.countryStats.proxies {
 		m.countryStats.proxies[pType] = make(map[string]bool)
@@ -231,6 +253,14 @@ func binCount(count uint) uint {
 	return uint((math.Ceil(float64(count) / 8)) * 8)
 }
 
+func sumMapValues(m *map[RendezvousMethod]uint) uint {
+	var s uint = 0
+	for _, v := range *m {
+		s += v
+	}
+	return s
+}
+
 type PromMetrics struct {
 	registry         *prometheus.Registry
 	ProxyTotal       *prometheus.CounterVec
@@ -310,7 +340,7 @@ func initPrometheus() *PromMetrics {
 			Name:      "rounded_client_poll_total",
 			Help:      "The number of snowflake client polls, rounded up to a multiple of 8",
 		},
-		[]string{"nat", "status"},
+		[]string{"nat", "status", "rendezvous_method"},
 	)
 
 	// We need to register our metrics so they can be exported.
diff --git a/broker/snowflake-broker_test.go b/broker/snowflake-broker_test.go
index e335258..d894145 100644
--- a/broker/snowflake-broker_test.go
+++ b/broker/snowflake-broker_test.go
@@ -89,7 +89,8 @@ func TestBroker(t *testing.T) {
 	copy(defaultBridge[:], defaultBridgeValue)
 
 	Convey("Context", t, func() {
-		ctx := NewBrokerContext(NullLogger())
+		buf := new(bytes.Buffer)
+		ctx := NewBrokerContext(log.New(buf, "", 0))
 		i := &IPC{ctx}
 
 		Convey("Adds Snowflake", func() {
@@ -149,6 +150,26 @@ func TestBroker(t *testing.T) {
 				clientOffers(i, w, r)
 				So(w.Code, ShouldEqual, http.StatusOK)
 				So(w.Body.String(), ShouldEqual, `{"error":"no snowflake proxies currently available"}`)
+
+				// Ensure that denial is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 8
+client-restricted-denied-count 8
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 8
+client-http-restricted-denied-count 8
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("with a proxy answer if available.", func() {
@@ -165,6 +186,26 @@ func TestBroker(t *testing.T) {
 				<-done
 				So(w.Body.String(), ShouldEqual, `{"answer":"test answer"}`)
 				So(w.Code, ShouldEqual, http.StatusOK)
+
+				// Ensure that match is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 8
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 8
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("with unrestricted proxy to unrestricted client if there are no restricted proxies", func() {
@@ -230,6 +271,26 @@ func TestBroker(t *testing.T) {
 				clientOffers(i, w, r)
 				So(w.Code, ShouldEqual, http.StatusServiceUnavailable)
 				So(w.Body.String(), ShouldEqual, "")
+
+				// Ensure that denial is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 8
+client-restricted-denied-count 8
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 8
+client-http-restricted-denied-count 8
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("with a proxy answer if available.", func() {
@@ -246,6 +307,26 @@ func TestBroker(t *testing.T) {
 				<-done
 				So(w.Body.String(), ShouldEqual, "fake answer")
 				So(w.Code, ShouldEqual, http.StatusOK)
+
+				// Ensure that match is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 8
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 8
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("Times out when no proxy responds.", func() {
@@ -288,6 +369,26 @@ func TestBroker(t *testing.T) {
 				body, err := decodeAMPArmorToString(w.Body)
 				So(err, ShouldBeNil)
 				So(body, ShouldEqual, `{"error":"no snowflake proxies currently available"}`)
+
+				// Ensure that denial is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 8
+client-restricted-denied-count 8
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 8
+client-ampcache-restricted-denied-count 8
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("with a proxy answer if available.", func() {
@@ -306,6 +407,26 @@ func TestBroker(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(body, ShouldEqual, `{"answer":"fake answer"}`)
 				So(w.Code, ShouldEqual, http.StatusOK)
+
+				// Ensure that match is correctly recorded in metrics
+				ctx.metrics.printMetrics()
+				So(buf.String(), ShouldContainSubstring, `client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 8
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 8
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 			})
 
 			Convey("Times out when no proxy responds.", func() {
@@ -652,7 +773,31 @@ func TestMetrics(t *testing.T) {
 			So(metricsStr, ShouldContainSubstring, "\nsnowflake-ips-standalone 1\n")
 			So(metricsStr, ShouldContainSubstring, "\nsnowflake-ips-badge 1\n")
 			So(metricsStr, ShouldContainSubstring, "\nsnowflake-ips-webext 1\n")
-			So(metricsStr, ShouldEndWith, "\nsnowflake-ips-total 4\nsnowflake-idle-count 8\nsnowflake-proxy-poll-with-relay-url-count 0\nsnowflake-proxy-poll-without-relay-url-count 8\nsnowflake-proxy-rejected-for-relay-url-count 0\nclient-denied-count 0\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 0\nsnowflake-ips-nat-restricted 0\nsnowflake-ips-nat-unrestricted 0\nsnowflake-ips-nat-unknown 1\n")
+			So(metricsStr, ShouldEndWith, `snowflake-ips-total 4
+snowflake-idle-count 8
+snowflake-proxy-poll-with-relay-url-count 0
+snowflake-proxy-poll-without-relay-url-count 8
+snowflake-proxy-rejected-for-relay-url-count 0
+client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+snowflake-ips-nat-restricted 0
+snowflake-ips-nat-unrestricted 0
+snowflake-ips-nat-unknown 1
+`)
 		})
 
 		//Test addition of client failures
@@ -666,7 +811,23 @@ func TestMetrics(t *testing.T) {
 			clientOffers(i, w, r)
 
 			ctx.metrics.printMetrics()
-			So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 8\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 0")
+			So(buf.String(), ShouldContainSubstring, `client-denied-count 8
+client-restricted-denied-count 8
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 8
+client-http-restricted-denied-count 8
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+`)
 
 			// Test reset
 			buf.Reset()
@@ -676,7 +837,31 @@ func TestMetrics(t *testing.T) {
 			So(buf.String(), ShouldContainSubstring, "\nsnowflake-ips-standalone 0\n")
 			So(buf.String(), ShouldContainSubstring, "\nsnowflake-ips-badge 0\n")
 			So(buf.String(), ShouldContainSubstring, "\nsnowflake-ips-webext 0\n")
-			So(buf.String(), ShouldContainSubstring, "\nsnowflake-ips-total 0\nsnowflake-idle-count 0\nsnowflake-proxy-poll-with-relay-url-count 0\nsnowflake-proxy-poll-without-relay-url-count 0\nsnowflake-proxy-rejected-for-relay-url-count 0\nclient-denied-count 0\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 0\nsnowflake-ips-nat-restricted 0\nsnowflake-ips-nat-unrestricted 0\nsnowflake-ips-nat-unknown 0\n")
+			So(buf.String(), ShouldContainSubstring, `snowflake-ips-total 0
+snowflake-idle-count 0
+snowflake-proxy-poll-with-relay-url-count 0
+snowflake-proxy-poll-without-relay-url-count 0
+snowflake-proxy-rejected-for-relay-url-count 0
+client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 0
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 0
+snowflake-ips-nat-restricted 0
+snowflake-ips-nat-unrestricted 0
+snowflake-ips-nat-unknown 0
+`)
 		})
 		//Test addition of client matches
 		Convey("for client-proxy match", func() {
diff --git a/broker/sqs.go b/broker/sqs.go
index 47b97c2..59fa4c7 100644
--- a/broker/sqs.go
+++ b/broker/sqs.go
@@ -148,7 +148,7 @@ func (r *sqsHandler) handleMessage(context context.Context, message *types.Messa
 		Body:       encPollReq,
 		RemoteAddr: "",
 	}
-	err = r.IPC.ClientOffers(arg, &response)
+	err = r.IPC.ClientOffers(arg, &response, RendezvousSqs)
 
 	if err != nil {
 		log.Printf("SQSHandler: error encountered when handling message: %v\n", err)
diff --git a/broker/sqs_test.go b/broker/sqs_test.go
index 0596888..0ffb778 100644
--- a/broker/sqs_test.go
+++ b/broker/sqs_test.go
@@ -21,8 +21,9 @@ import (
 func TestSQS(t *testing.T) {
 
 	Convey("Context", t, func() {
-		ctx := NewBrokerContext(NullLogger())
-		i := &IPC{ctx}
+		buf := new(bytes.Buffer)
+		ipcCtx := NewBrokerContext(log.New(buf, "", 0))
+		i := &IPC{ipcCtx}
 
 		var logBuffer bytes.Buffer
 		log.SetOutput(&logBuffer)
@@ -187,6 +188,25 @@ func TestSQS(t *testing.T) {
 							numTimes += 1
 							if numTimes == 1 {
 								c.So(input.MessageBody, ShouldEqual, aws.String("{\"answer\":\"fake answer\"}"))
+								// Ensure that match is correctly recorded in metrics
+								ipcCtx.metrics.printMetrics()
+								c.So(buf.String(), ShouldContainSubstring, `client-denied-count 0
+client-restricted-denied-count 0
+client-unrestricted-denied-count 0
+client-snowflake-match-count 8
+client-http-denied-count 0
+client-http-restricted-denied-count 0
+client-http-unrestricted-denied-count 0
+client-snowflake-http-match-count 0
+client-ampcache-denied-count 0
+client-ampcache-restricted-denied-count 0
+client-ampcache-unrestricted-denied-count 0
+client-snowflake-ampcache-match-count 0
+client-sqs-denied-count 0
+client-sqs-restricted-denied-count 0
+client-sqs-unrestricted-denied-count 0
+client-snowflake-sqs-match-count 8
+`)
 								wg.Done()
 							}
 							return &sqs.SendMessageOutput{}, nil
@@ -194,7 +214,7 @@ func TestSQS(t *testing.T) {
 					)
 					runSQSHandler(sqsHandlerContext)
 
-					snowflake := ctx.AddSnowflake("fake", "", NATUnrestricted, 0)
+					snowflake := ipcCtx.AddSnowflake("fake", "", NATUnrestricted, 0)
 
 					offer := <-snowflake.offerChannel
 					So(offer.sdp, ShouldResemble, []byte("fake"))
diff --git a/doc/broker-spec.txt b/doc/broker-spec.txt
index b138605..e6b51b1 100644
--- a/doc/broker-spec.txt
+++ b/doc/broker-spec.txt
@@ -82,6 +82,96 @@ Metrics data from the Snowflake broker can be retrieved by sending an HTTP GET r
         A count of the number of times a client successfully received a
         proxy from the broker, rounded up to the nearest multiple of 8.
 
+    "client-http-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client has requested a proxy using
+        the HTTP rendezvous method from the broker but no proxies were 
+        available, rounded up to the nearest multiple of 8.
+
+    "client-http-restricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with a restricted or
+        unknown NAT type has requested a proxy using the HTTP rendezvous 
+        method from the broker but no proxies were available, rounded up to 
+        the nearest multiple of 8.
+
+    "client-http-unrestricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with an unrestricted NAT
+        type has requested a proxy using the HTTP rendezvous method from 
+        the broker but no proxies were available, rounded up to the nearest 
+        multiple of 8.
+
+    "client-snowflake-http-match-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client successfully received a
+        proxy using the HTTP rendezvous method from the broker, rounded 
+        up to the nearest multiple of 8.
+    
+    "client-ampcache-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client has requested a proxy using
+        the ampcache rendezvous method from the broker but no proxies were 
+        available, rounded up to the nearest multiple of 8.
+
+    "client-ampcache-restricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with a restricted or
+        unknown NAT type has requested a proxy using the ampcache rendezvous 
+        method from the broker but no proxies were available, rounded up to 
+        the nearest multiple of 8.
+
+    "client-ampcache-unrestricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with an unrestricted NAT
+        type has requested a proxy using the ampcache rendezvous method from 
+        the broker but no proxies were available, rounded up to the nearest 
+        multiple of 8.
+
+    "client-snowflake-ampcache-match-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client successfully received a
+        proxy using the ampcache rendezvous method from the broker, rounded 
+        up to the nearest multiple of 8.
+    
+    "client-sqs-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client has requested a proxy using
+        the sqs rendezvous method from the broker but no proxies were 
+        available, rounded up to the nearest multiple of 8.
+
+    "client-sqs-restricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with a restricted or
+        unknown NAT type has requested a proxy using the sqs rendezvous 
+        method from the broker but no proxies were available, rounded up to 
+        the nearest multiple of 8.
+
+    "client-sqs-unrestricted-denied-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client with an unrestricted NAT
+        type has requested a proxy using the sqs rendezvous method from 
+        the broker but no proxies were available, rounded up to the nearest 
+        multiple of 8.
+
+    "client-snowflake-sqs-match-count" NUM NL
+        [At most once.]
+
+        A count of the number of times a client successfully received a
+        proxy using the sqs rendezvous method from the broker, rounded 
+        up to the nearest multiple of 8.
+
     "snowflake-ips-nat-restricted" NUM NL
         [At most once.]