557 lines
29 KiB
Go
557 lines
29 KiB
Go
/*
|
|
Copyright 2020 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package metrics
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
cmg "github.com/prometheus/client_model/go"
|
|
"github.com/prometheus/common/expfmt"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
)
|
|
|
|
var (
|
|
statusMap map[int]string = map[int]string{
|
|
0: "Success",
|
|
1: "Failure",
|
|
2: "Unknown",
|
|
}
|
|
)
|
|
|
|
const (
|
|
httpPattern = "/metrics"
|
|
addr = "localhost:0"
|
|
)
|
|
|
|
type fakeOpStatus struct {
|
|
statusCode int
|
|
}
|
|
|
|
func (s *fakeOpStatus) String() string {
|
|
if str, ok := statusMap[s.statusCode]; ok {
|
|
return str
|
|
}
|
|
return "Unknown"
|
|
}
|
|
|
|
func initMgr() (MetricsManager, *sync.WaitGroup, *http.Server) {
|
|
wg := &sync.WaitGroup{}
|
|
wg.Add(1)
|
|
mgr := NewMetricsManager()
|
|
srv, err := mgr.StartMetricsEndpoint(httpPattern, addr, nil, wg)
|
|
if err != nil {
|
|
log.Fatalf("failed to start serving [%v]", err)
|
|
}
|
|
return mgr, wg, srv
|
|
}
|
|
|
|
func shutdown(srv *http.Server, wg *sync.WaitGroup) {
|
|
if err := srv.Shutdown(context.Background()); err != nil {
|
|
panic(err)
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestNew(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
defer shutdown(srv, wg)
|
|
if mgr == nil {
|
|
t.Errorf("failed testing new")
|
|
}
|
|
}
|
|
|
|
func TestDropNonExistingOperation(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
defer shutdown(srv, wg)
|
|
op := Operation{
|
|
Name: "drop-non-existing-operation-should-be-noop",
|
|
Driver: "driver",
|
|
ResourceID: types.UID("uid"),
|
|
}
|
|
mgr.DropOperation(op)
|
|
}
|
|
|
|
func TestRecordMetricsForNonExistingOperation(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
srvAddr := "http://" + srv.Addr + httpPattern
|
|
defer shutdown(srv, wg)
|
|
op := Operation{
|
|
Name: "no-metrics-should-be-recorded-as-operation-did-not-start",
|
|
Driver: "driver",
|
|
ResourceID: types.UID("uid"),
|
|
}
|
|
mgr.RecordMetrics(op, nil)
|
|
rsp, err := http.Get(srvAddr)
|
|
if err != nil || rsp.StatusCode != http.StatusOK {
|
|
t.Errorf("failed to get response from server %v, %v", err, rsp)
|
|
}
|
|
r, err := ioutil.ReadAll(rsp.Body)
|
|
if err != nil {
|
|
t.Errorf("failed to read response body %v", err)
|
|
}
|
|
if strings.Contains(string(r), op.Name) {
|
|
t.Errorf("found metric should have been dropped for operation [%s] [%s]", op.Name, string(r))
|
|
}
|
|
}
|
|
|
|
func TestDropOperation(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
srvAddr := "http://" + srv.Addr + httpPattern
|
|
defer shutdown(srv, wg)
|
|
op := Operation{
|
|
Name: "should-have-been-dropped",
|
|
Driver: "driver",
|
|
ResourceID: types.UID("uid"),
|
|
}
|
|
mgr.OperationStart(op)
|
|
mgr.DropOperation(op)
|
|
time.Sleep(300 * time.Millisecond)
|
|
rsp, err := http.Get(srvAddr)
|
|
if err != nil || rsp.StatusCode != http.StatusOK {
|
|
t.Errorf("failed to get response from server %v, %v", err, rsp)
|
|
}
|
|
r, err := ioutil.ReadAll(rsp.Body)
|
|
if err != nil {
|
|
t.Errorf("failed to read response body %v", err)
|
|
}
|
|
if strings.Contains(string(r), op.Name) {
|
|
t.Errorf("found metric should have been dropped for operation [%s] [%s]", op.Name, string(r))
|
|
}
|
|
// re-add with a different name
|
|
op.Name = "should-have-been-added"
|
|
mgr.OperationStart(op)
|
|
time.Sleep(300 * time.Millisecond)
|
|
opStatus := &fakeOpStatus{
|
|
statusCode: 0,
|
|
}
|
|
mgr.RecordMetrics(op, opStatus)
|
|
expected :=
|
|
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
|
# TYPE snapshot_controller_operation_total_seconds histogram
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.25"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="should-have-been-added",operation_status="Success"} 0.3
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="should-have-been-added",operation_status="Success"} 1
|
|
`
|
|
|
|
if err := verifyMetric(expected, srvAddr); err != nil {
|
|
t.Errorf("failed testing[%v]", err)
|
|
}
|
|
}
|
|
|
|
func TestUnknownStatus(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
srvAddr := "http://" + srv.Addr + httpPattern
|
|
defer shutdown(srv, wg)
|
|
op := Operation{
|
|
Name: "unknown-status-operation",
|
|
Driver: "driver",
|
|
ResourceID: types.UID("uid"),
|
|
}
|
|
mgr.OperationStart(op)
|
|
// should create a Unknown data point with latency ~300ms
|
|
time.Sleep(300 * time.Millisecond)
|
|
mgr.RecordMetrics(op, nil)
|
|
expected :=
|
|
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
|
# TYPE snapshot_controller_operation_total_seconds histogram
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.25"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown"} 0.3
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown"} 1
|
|
`
|
|
|
|
if err := verifyMetric(expected, srvAddr); err != nil {
|
|
t.Errorf("failed testing[%v]", err)
|
|
}
|
|
}
|
|
|
|
func TestRecordMetrics(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
srvAddr := "http://" + srv.Addr + httpPattern
|
|
defer shutdown(srv, wg)
|
|
// add an operation
|
|
op := Operation{
|
|
Name: "op1",
|
|
Driver: "driver1",
|
|
ResourceID: types.UID("uid1"),
|
|
}
|
|
mgr.OperationStart(op)
|
|
// should create a Success data point with latency ~ 1100ms
|
|
time.Sleep(1100 * time.Millisecond)
|
|
success := &fakeOpStatus{
|
|
statusCode: 0,
|
|
}
|
|
mgr.RecordMetrics(op, success)
|
|
|
|
// add another operation metric
|
|
op.Name = "op2"
|
|
op.Driver = "driver2"
|
|
op.ResourceID = types.UID("uid2")
|
|
mgr.OperationStart(op)
|
|
// should create a Failure data point with latency ~ 100ms
|
|
time.Sleep(100 * time.Millisecond)
|
|
failure := &fakeOpStatus{
|
|
statusCode: 1,
|
|
}
|
|
mgr.RecordMetrics(op, failure)
|
|
|
|
expected :=
|
|
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
|
# TYPE snapshot_controller_operation_total_seconds histogram
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.25"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.5"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="op1",operation_status="Success"} 1.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="op1",operation_status="Success"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="op2",operation_status="Failure"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="op2",operation_status="Failure"} 1
|
|
`
|
|
if err := verifyMetric(expected, srvAddr); err != nil {
|
|
t.Errorf("failed testing [%v]", err)
|
|
}
|
|
}
|
|
|
|
func TestConcurrency(t *testing.T) {
|
|
mgr, wg, srv := initMgr()
|
|
srvAddr := "http://" + srv.Addr + httpPattern
|
|
defer shutdown(srv, wg)
|
|
success := &fakeOpStatus{
|
|
statusCode: 0,
|
|
}
|
|
failure := &fakeOpStatus{
|
|
statusCode: 1,
|
|
}
|
|
ops := []struct {
|
|
op Operation
|
|
desiredLatencyMs time.Duration
|
|
status OperationStatus
|
|
drop bool
|
|
}{
|
|
{
|
|
Operation{
|
|
Name: "success1",
|
|
Driver: "driver1",
|
|
ResourceID: types.UID("uid1"),
|
|
},
|
|
100,
|
|
success,
|
|
false,
|
|
},
|
|
{
|
|
Operation{
|
|
Name: "success2",
|
|
Driver: "driver2",
|
|
ResourceID: types.UID("uid2"),
|
|
},
|
|
100,
|
|
success,
|
|
false,
|
|
},
|
|
{
|
|
Operation{
|
|
Name: "failure1",
|
|
Driver: "driver3",
|
|
ResourceID: types.UID("uid3"),
|
|
},
|
|
100,
|
|
failure,
|
|
false,
|
|
},
|
|
{
|
|
Operation{
|
|
Name: "failure2",
|
|
Driver: "driver4",
|
|
ResourceID: types.UID("uid4"),
|
|
},
|
|
100,
|
|
failure,
|
|
false,
|
|
},
|
|
{
|
|
Operation{
|
|
Name: "unknown",
|
|
Driver: "driver5",
|
|
ResourceID: types.UID("uid5"),
|
|
},
|
|
100,
|
|
nil,
|
|
false,
|
|
},
|
|
{
|
|
Operation{
|
|
Name: "drop-from-cache",
|
|
Driver: "driver6",
|
|
ResourceID: types.UID("uid6"),
|
|
},
|
|
100,
|
|
nil,
|
|
true,
|
|
},
|
|
}
|
|
|
|
for i := range ops {
|
|
mgr.OperationStart(ops[i].op)
|
|
}
|
|
// add an extra operation which should remain in the cache
|
|
remaining := Operation{
|
|
Name: "remaining-in-cache",
|
|
Driver: "driver7",
|
|
ResourceID: types.UID("uid7"),
|
|
}
|
|
mgr.OperationStart(remaining)
|
|
|
|
var wgMetrics sync.WaitGroup
|
|
|
|
for i := range ops {
|
|
wgMetrics.Add(1)
|
|
go func(i int) {
|
|
defer wgMetrics.Done()
|
|
if ops[i].desiredLatencyMs > 0 {
|
|
time.Sleep(ops[i].desiredLatencyMs * time.Millisecond)
|
|
}
|
|
if ops[i].drop {
|
|
mgr.DropOperation(ops[i].op)
|
|
} else {
|
|
mgr.RecordMetrics(ops[i].op, ops[i].status)
|
|
}
|
|
}(i)
|
|
}
|
|
wgMetrics.Wait()
|
|
|
|
// validate
|
|
expected :=
|
|
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
|
# TYPE snapshot_controller_operation_total_seconds histogram
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="success1",operation_status="Success"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="success1",operation_status="Success"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="success2",operation_status="Success"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="success2",operation_status="Success"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver3",operation_name="failure1",operation_status="Failure"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver3",operation_name="failure1",operation_status="Failure"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver4",operation_name="failure2",operation_status="Failure"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver4",operation_name="failure2",operation_status="Failure"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.1"} 0
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.25"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="1"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="2.5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="5"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="10"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="15"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="30"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="60"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="120"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="300"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="600"} 1
|
|
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="+Inf"} 1
|
|
snapshot_controller_operation_total_seconds_sum{driver_name="driver5",operation_name="unknown",operation_status="Unknown"} 0.1
|
|
snapshot_controller_operation_total_seconds_count{driver_name="driver5",operation_name="unknown",operation_status="Unknown"} 1
|
|
`
|
|
if err := verifyMetric(expected, srvAddr); err != nil {
|
|
t.Errorf("failed testing [%v]", err)
|
|
}
|
|
}
|
|
|
|
func verifyMetric(expected, srvAddr string) error {
|
|
rsp, err := http.Get(srvAddr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if rsp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("failed to get response from serve: %s", http.StatusText(rsp.StatusCode))
|
|
}
|
|
r, err := ioutil.ReadAll(rsp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
format := expfmt.ResponseFormat(rsp.Header)
|
|
reader := strings.NewReader(string(r))
|
|
decoder := expfmt.NewDecoder(reader, format)
|
|
mf := &cmg.MetricFamily{}
|
|
if err := decoder.Decode(mf); err != nil {
|
|
return err
|
|
}
|
|
reader = strings.NewReader(expected)
|
|
decoder = expfmt.NewDecoder(reader, format)
|
|
expectedMf := &cmg.MetricFamily{}
|
|
if err := decoder.Decode(expectedMf); err != nil {
|
|
return err
|
|
}
|
|
if !containsMetrics(expectedMf, mf) {
|
|
return fmt.Errorf("failed testing, expected\n%s\n, got\n%s\n", expected, string(r))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func containsMetrics(expected, got *cmg.MetricFamily) bool {
|
|
if (got.Name == nil || *(got.Name) != *(expected.Name)) ||
|
|
(got.Type == nil || *(got.Type) != *(expected.Type)) ||
|
|
(got.Help == nil || *(got.Help) != *(expected.Help)) {
|
|
return false
|
|
}
|
|
|
|
numRecords := len(expected.Metric)
|
|
if len(got.Metric) < numRecords {
|
|
return false
|
|
}
|
|
matchCount := 0
|
|
for i := 0; i < len(got.Metric); i++ {
|
|
for j := 0; j < numRecords; j++ {
|
|
// labels should be the same
|
|
if !reflect.DeepEqual(got.Metric[i].Label, expected.Metric[j].Label) {
|
|
continue
|
|
}
|
|
gh := got.Metric[i].Histogram
|
|
eh := expected.Metric[j].Histogram
|
|
if gh == nil {
|
|
continue
|
|
}
|
|
if !reflect.DeepEqual(gh.Bucket, eh.Bucket) {
|
|
continue
|
|
}
|
|
// this is a sum record, expecting a latency which is more than the
|
|
// expected one. If the sum is smaller than expected, it will be considered
|
|
// as NOT a match
|
|
if gh.SampleSum == nil || *(gh.SampleSum) < *(eh.SampleSum) {
|
|
continue
|
|
}
|
|
if gh.SampleCount == nil || *(gh.SampleCount) != *(eh.SampleCount) {
|
|
continue
|
|
}
|
|
// this is a match
|
|
matchCount = matchCount + 1
|
|
break
|
|
}
|
|
}
|
|
return matchCount == numRecords
|
|
}
|