Add snapshot controller metrics
Signed-off-by: Grant Griffiths <grant@portworx.com>
This commit is contained in:
Grant Griffiths
@@ -19,10 +19,12 @@ package metrics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -86,9 +88,8 @@ func TestNew(t *testing.T) {
|
||||
func TestDropNonExistingOperation(t *testing.T) {
|
||||
mgr, wg, srv := initMgr()
|
||||
defer shutdown(srv, wg)
|
||||
op := Operation{
|
||||
op := OperationKey{
|
||||
Name: "drop-non-existing-operation-should-be-noop",
|
||||
Driver: "driver",
|
||||
ResourceID: types.UID("uid"),
|
||||
}
|
||||
mgr.DropOperation(op)
|
||||
@@ -98,12 +99,11 @@ func TestRecordMetricsForNonExistingOperation(t *testing.T) {
|
||||
mgr, wg, srv := initMgr()
|
||||
srvAddr := "http://" + srv.Addr + httpPattern
|
||||
defer shutdown(srv, wg)
|
||||
op := Operation{
|
||||
opKey := OperationKey{
|
||||
Name: "no-metrics-should-be-recorded-as-operation-did-not-start",
|
||||
Driver: "driver",
|
||||
ResourceID: types.UID("uid"),
|
||||
}
|
||||
mgr.RecordMetrics(op, nil)
|
||||
mgr.RecordMetrics(opKey, nil, "driver")
|
||||
rsp, err := http.Get(srvAddr)
|
||||
if err != nil || rsp.StatusCode != http.StatusOK {
|
||||
t.Errorf("failed to get response from server %v, %v", err, rsp)
|
||||
@@ -112,8 +112,8 @@ func TestRecordMetricsForNonExistingOperation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("failed to read response body %v", err)
|
||||
}
|
||||
if strings.Contains(string(r), op.Name) {
|
||||
t.Errorf("found metric should have been dropped for operation [%s] [%s]", op.Name, string(r))
|
||||
if strings.Contains(string(r), opKey.Name) {
|
||||
t.Errorf("found metric should have been dropped for operation [%s] [%s]", opKey.Name, string(r))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,13 +121,13 @@ func TestDropOperation(t *testing.T) {
|
||||
mgr, wg, srv := initMgr()
|
||||
srvAddr := "http://" + srv.Addr + httpPattern
|
||||
defer shutdown(srv, wg)
|
||||
op := Operation{
|
||||
opKey := OperationKey{
|
||||
Name: "should-have-been-dropped",
|
||||
Driver: "driver",
|
||||
ResourceID: types.UID("uid"),
|
||||
}
|
||||
mgr.OperationStart(op)
|
||||
mgr.DropOperation(op)
|
||||
opVal := NewOperationValue("driver", DynamicSnapshotType)
|
||||
mgr.OperationStart(opKey, opVal)
|
||||
mgr.DropOperation(opKey)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
rsp, err := http.Get(srvAddr)
|
||||
if err != nil || rsp.StatusCode != http.StatusOK {
|
||||
@@ -137,36 +137,36 @@ func TestDropOperation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("failed to read response body %v", err)
|
||||
}
|
||||
if strings.Contains(string(r), op.Name) {
|
||||
t.Errorf("found metric should have been dropped for operation [%s] [%s]", op.Name, string(r))
|
||||
if strings.Contains(string(r), opKey.Name) {
|
||||
t.Errorf("found metric should have been dropped for operation [%s] [%s]", opKey.Name, string(r))
|
||||
}
|
||||
// re-add with a different name
|
||||
op.Name = "should-have-been-added"
|
||||
mgr.OperationStart(op)
|
||||
opKey.Name = "should-have-been-added"
|
||||
mgr.OperationStart(opKey, opVal)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
opStatus := &fakeOpStatus{
|
||||
statusCode: 0,
|
||||
}
|
||||
mgr.RecordMetrics(op, opStatus)
|
||||
mgr.RecordMetrics(opKey, opStatus, "driver")
|
||||
expected :=
|
||||
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
||||
# TYPE snapshot_controller_operation_total_seconds histogram
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="should-have-been-added",operation_status="Success"} 0.3
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="should-have-been-added",operation_status="Success"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type=""} 0.3
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="should-have-been-added",operation_status="Success",snapshot_type=""} 1
|
||||
`
|
||||
|
||||
if err := verifyMetric(expected, srvAddr); err != nil {
|
||||
@@ -178,34 +178,33 @@ func TestUnknownStatus(t *testing.T) {
|
||||
mgr, wg, srv := initMgr()
|
||||
srvAddr := "http://" + srv.Addr + httpPattern
|
||||
defer shutdown(srv, wg)
|
||||
op := Operation{
|
||||
opKey := OperationKey{
|
||||
Name: "unknown-status-operation",
|
||||
Driver: "driver",
|
||||
ResourceID: types.UID("uid"),
|
||||
}
|
||||
mgr.OperationStart(op)
|
||||
mgr.OperationStart(opKey, NewOperationValue("driver", DynamicSnapshotType))
|
||||
// should create a Unknown data point with latency ~300ms
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
mgr.RecordMetrics(op, nil)
|
||||
mgr.RecordMetrics(opKey, nil, "driver")
|
||||
expected :=
|
||||
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
||||
# TYPE snapshot_controller_operation_total_seconds histogram
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown"} 0.3
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type=""} 0.3
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver",operation_name="unknown-status-operation",operation_status="Unknown",snapshot_type=""} 1
|
||||
`
|
||||
|
||||
if err := verifyMetric(expected, srvAddr); err != nil {
|
||||
@@ -218,66 +217,65 @@ func TestRecordMetrics(t *testing.T) {
|
||||
srvAddr := "http://" + srv.Addr + httpPattern
|
||||
defer shutdown(srv, wg)
|
||||
// add an operation
|
||||
op := Operation{
|
||||
opKey := OperationKey{
|
||||
Name: "op1",
|
||||
Driver: "driver1",
|
||||
ResourceID: types.UID("uid1"),
|
||||
}
|
||||
mgr.OperationStart(op)
|
||||
opVal := NewOperationValue("driver", DynamicSnapshotType)
|
||||
mgr.OperationStart(opKey, opVal)
|
||||
// should create a Success data point with latency ~ 1100ms
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
success := &fakeOpStatus{
|
||||
statusCode: 0,
|
||||
}
|
||||
mgr.RecordMetrics(op, success)
|
||||
mgr.RecordMetrics(opKey, success, "driver")
|
||||
|
||||
// add another operation metric
|
||||
op.Name = "op2"
|
||||
op.Driver = "driver2"
|
||||
op.ResourceID = types.UID("uid2")
|
||||
mgr.OperationStart(op)
|
||||
opKey.Name = "op2"
|
||||
opKey.ResourceID = types.UID("uid2")
|
||||
mgr.OperationStart(opKey, opVal)
|
||||
// should create a Failure data point with latency ~ 100ms
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
failure := &fakeOpStatus{
|
||||
statusCode: 1,
|
||||
}
|
||||
mgr.RecordMetrics(op, failure)
|
||||
mgr.RecordMetrics(opKey, failure, "driver")
|
||||
|
||||
expected :=
|
||||
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
||||
# TYPE snapshot_controller_operation_total_seconds histogram
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="0.5"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="op1",operation_status="Success"} 1.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="op1",operation_status="Success"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="op2",operation_status="Failure"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="op2",operation_status="Failure"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="0.25"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="0.5"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type=""} 1.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="op1",operation_status="Success",snapshot_type=""} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="op2",operation_status="Failure",snapshot_type=""} 1
|
||||
`
|
||||
if err := verifyMetric(expected, srvAddr); err != nil {
|
||||
t.Errorf("failed testing [%v]", err)
|
||||
@@ -295,15 +293,14 @@ func TestConcurrency(t *testing.T) {
|
||||
statusCode: 1,
|
||||
}
|
||||
ops := []struct {
|
||||
op Operation
|
||||
op OperationKey
|
||||
desiredLatencyMs time.Duration
|
||||
status OperationStatus
|
||||
drop bool
|
||||
}{
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "success1",
|
||||
Driver: "driver1",
|
||||
ResourceID: types.UID("uid1"),
|
||||
},
|
||||
100,
|
||||
@@ -311,9 +308,8 @@ func TestConcurrency(t *testing.T) {
|
||||
false,
|
||||
},
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "success2",
|
||||
Driver: "driver2",
|
||||
ResourceID: types.UID("uid2"),
|
||||
},
|
||||
100,
|
||||
@@ -321,9 +317,8 @@ func TestConcurrency(t *testing.T) {
|
||||
false,
|
||||
},
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "failure1",
|
||||
Driver: "driver3",
|
||||
ResourceID: types.UID("uid3"),
|
||||
},
|
||||
100,
|
||||
@@ -331,9 +326,8 @@ func TestConcurrency(t *testing.T) {
|
||||
false,
|
||||
},
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "failure2",
|
||||
Driver: "driver4",
|
||||
ResourceID: types.UID("uid4"),
|
||||
},
|
||||
100,
|
||||
@@ -341,9 +335,8 @@ func TestConcurrency(t *testing.T) {
|
||||
false,
|
||||
},
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "unknown",
|
||||
Driver: "driver5",
|
||||
ResourceID: types.UID("uid5"),
|
||||
},
|
||||
100,
|
||||
@@ -351,9 +344,8 @@ func TestConcurrency(t *testing.T) {
|
||||
false,
|
||||
},
|
||||
{
|
||||
Operation{
|
||||
OperationKey{
|
||||
Name: "drop-from-cache",
|
||||
Driver: "driver6",
|
||||
ResourceID: types.UID("uid6"),
|
||||
},
|
||||
100,
|
||||
@@ -363,15 +355,20 @@ func TestConcurrency(t *testing.T) {
|
||||
}
|
||||
|
||||
for i := range ops {
|
||||
mgr.OperationStart(ops[i].op)
|
||||
mgr.OperationStart(ops[i].op, OperationValue{
|
||||
Driver: fmt.Sprintf("driver%v", i),
|
||||
SnapshotType: string(DynamicSnapshotType),
|
||||
})
|
||||
}
|
||||
// add an extra operation which should remain in the cache
|
||||
remaining := Operation{
|
||||
remaining := OperationKey{
|
||||
Name: "remaining-in-cache",
|
||||
Driver: "driver7",
|
||||
ResourceID: types.UID("uid7"),
|
||||
}
|
||||
mgr.OperationStart(remaining)
|
||||
mgr.OperationStart(remaining, OperationValue{
|
||||
Driver: "driver7",
|
||||
SnapshotType: string(DynamicSnapshotType),
|
||||
})
|
||||
|
||||
var wgMetrics sync.WaitGroup
|
||||
|
||||
@@ -385,7 +382,7 @@ func TestConcurrency(t *testing.T) {
|
||||
if ops[i].drop {
|
||||
mgr.DropOperation(ops[i].op)
|
||||
} else {
|
||||
mgr.RecordMetrics(ops[i].op, ops[i].status)
|
||||
mgr.RecordMetrics(ops[i].op, ops[i].status, fmt.Sprintf("driver%v", i))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
@@ -395,86 +392,86 @@ func TestConcurrency(t *testing.T) {
|
||||
expected :=
|
||||
`# HELP snapshot_controller_operation_total_seconds [ALPHA] Total number of seconds spent by the controller on an operation from end to end
|
||||
# TYPE snapshot_controller_operation_total_seconds histogram
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="success1",operation_status="Success"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="success1",operation_status="Success"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="success2",operation_status="Success"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="success2",operation_status="Success"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver3",operation_name="failure1",operation_status="Failure"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver3",operation_name="failure1",operation_status="Failure"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver4",operation_name="failure2",operation_status="Failure"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver4",operation_name="failure2",operation_status="Failure"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver5",operation_name="unknown",operation_status="Unknown"} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver5",operation_name="unknown",operation_status="Unknown"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver1",operation_name="success1",operation_status="Success",snapshot_type=""} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver2",operation_name="success2",operation_status="Success",snapshot_type=""} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver3",operation_name="failure1",operation_status="Failure",snapshot_type=""} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver4",operation_name="failure2",operation_status="Failure",snapshot_type=""} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="0.1"} 0
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="0.25"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="0.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="1"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="2.5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="5"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="10"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="15"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="30"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="60"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="120"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="300"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="600"} 1
|
||||
snapshot_controller_operation_total_seconds_bucket{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type="",le="+Inf"} 1
|
||||
snapshot_controller_operation_total_seconds_sum{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type=""} 0.1
|
||||
snapshot_controller_operation_total_seconds_count{driver_name="driver5",operation_name="unknown",operation_status="Unknown",snapshot_type=""} 1
|
||||
`
|
||||
if err := verifyMetric(expected, srvAddr); err != nil {
|
||||
t.Errorf("failed testing [%v]", err)
|
||||
@@ -494,63 +491,141 @@ func verifyMetric(expected, srvAddr string) error {
|
||||
return err
|
||||
}
|
||||
format := expfmt.ResponseFormat(rsp.Header)
|
||||
reader := strings.NewReader(string(r))
|
||||
decoder := expfmt.NewDecoder(reader, format)
|
||||
mf := &cmg.MetricFamily{}
|
||||
if err := decoder.Decode(mf); err != nil {
|
||||
return err
|
||||
gotReader := strings.NewReader(string(r))
|
||||
gotDecoder := expfmt.NewDecoder(gotReader, format)
|
||||
expectedReader := strings.NewReader(expected)
|
||||
expectedDecoder := expfmt.NewDecoder(expectedReader, format)
|
||||
|
||||
gotMfs := []*cmg.MetricFamily{}
|
||||
expectedMfs := []*cmg.MetricFamily{}
|
||||
for {
|
||||
gotMf := &cmg.MetricFamily{}
|
||||
gotErr := gotDecoder.Decode(gotMf)
|
||||
expectedMf := &cmg.MetricFamily{}
|
||||
if expectedErr := expectedDecoder.Decode(expectedMf); expectedErr != nil {
|
||||
// return correctly if both are EOF
|
||||
if expectedErr == io.EOF && gotErr == io.EOF {
|
||||
break
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
gotMfs = append(gotMfs, gotMf)
|
||||
expectedMfs = append(expectedMfs, expectedMf)
|
||||
}
|
||||
reader = strings.NewReader(expected)
|
||||
decoder = expfmt.NewDecoder(reader, format)
|
||||
expectedMf := &cmg.MetricFamily{}
|
||||
if err := decoder.Decode(expectedMf); err != nil {
|
||||
return err
|
||||
}
|
||||
if !containsMetrics(expectedMf, mf) {
|
||||
|
||||
if !containsMetrics(expectedMfs, gotMfs) {
|
||||
return fmt.Errorf("failed testing, expected\n%s\n, got\n%s\n", expected, string(r))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsMetrics(expected, got *cmg.MetricFamily) bool {
|
||||
if (got.Name == nil || *(got.Name) != *(expected.Name)) ||
|
||||
(got.Type == nil || *(got.Type) != *(expected.Type)) ||
|
||||
(got.Help == nil || *(got.Help) != *(expected.Help)) {
|
||||
// sortMfs, sorts metric families in alphabetical order by type.
|
||||
// currently only supports counter and histogram
|
||||
func sortMfs(mfs []*cmg.MetricFamily) []*cmg.MetricFamily {
|
||||
var sortedMfs []*cmg.MetricFamily
|
||||
|
||||
// Sort first by type
|
||||
sort.Slice(mfs, func(i, j int) bool {
|
||||
return *mfs[i].Type < *mfs[j].Type
|
||||
})
|
||||
|
||||
// Next, sort by length of name
|
||||
sort.Slice(mfs, func(i, j int) bool {
|
||||
return len(*mfs[i].Name) < len(*mfs[j].Name)
|
||||
})
|
||||
|
||||
return sortedMfs
|
||||
}
|
||||
|
||||
func containsMetrics(expectedMfs, gotMfs []*cmg.MetricFamily) bool {
|
||||
if len(gotMfs) != len(expectedMfs) {
|
||||
fmt.Printf("Not same length: expected and got metrics families: %v vs. %v\n", len(expectedMfs), len(gotMfs))
|
||||
return false
|
||||
}
|
||||
|
||||
numRecords := len(expected.Metric)
|
||||
if len(got.Metric) < numRecords {
|
||||
return false
|
||||
}
|
||||
matchCount := 0
|
||||
for i := 0; i < len(got.Metric); i++ {
|
||||
for j := 0; j < numRecords; j++ {
|
||||
// labels should be the same
|
||||
if !reflect.DeepEqual(got.Metric[i].Label, expected.Metric[j].Label) {
|
||||
continue
|
||||
// sort metric families for deterministic comparison.
|
||||
sortedExpectedMfs := sortMfs(expectedMfs)
|
||||
sortedGotMfs := sortMfs(gotMfs)
|
||||
|
||||
// compare expected vs. sorted actual metrics
|
||||
for k, got := range sortedGotMfs {
|
||||
matchCount := 0
|
||||
expected := sortedExpectedMfs[k]
|
||||
|
||||
if (got.Name == nil || *(got.Name) != *(expected.Name)) ||
|
||||
(got.Type == nil || *(got.Type) != *(expected.Type)) ||
|
||||
(got.Help == nil || *(got.Help) != *(expected.Help)) {
|
||||
fmt.Printf("invalid header info: got: %v, expected: %v\n", *got.Name, *expected.Name)
|
||||
fmt.Printf("invalid header info: got: %v, expected: %v\n", *got.Type, *expected.Type)
|
||||
fmt.Printf("invalid header info: got: %v, expected: %v\n", *got.Help, *expected.Help)
|
||||
return false
|
||||
}
|
||||
|
||||
numRecords := len(expected.Metric)
|
||||
if len(got.Metric) < numRecords {
|
||||
fmt.Printf("Not the same number of records: got.Metric: %v, numRecords: %v\n", len(got.Metric), numRecords)
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(got.Metric); i++ {
|
||||
for j := 0; j < numRecords; j++ {
|
||||
if got.Metric[i].Histogram == nil && expected.Metric[j].Histogram != nil ||
|
||||
got.Metric[i].Histogram != nil && expected.Metric[j].Histogram == nil {
|
||||
fmt.Printf("got metric and expected metric histogram type mismatch")
|
||||
return false
|
||||
}
|
||||
|
||||
// labels should be the same
|
||||
if !reflect.DeepEqual(got.Metric[i].Label, expected.Metric[j].Label) {
|
||||
continue
|
||||
}
|
||||
|
||||
// metric type specific checks
|
||||
switch {
|
||||
case got.Metric[i].Histogram != nil && expected.Metric[j].Histogram != nil:
|
||||
gh := got.Metric[i].Histogram
|
||||
eh := expected.Metric[j].Histogram
|
||||
if gh == nil || eh == nil {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(gh.Bucket, eh.Bucket) {
|
||||
fmt.Println("got and expected histogram bucket not equal")
|
||||
continue
|
||||
}
|
||||
|
||||
// this is a sum record, expecting a latency which is more than the
|
||||
// expected one. If the sum is smaller than expected, it will be considered
|
||||
// as NOT a match
|
||||
if gh.SampleSum == nil || eh.SampleSum == nil || *(gh.SampleSum) < *(eh.SampleSum) {
|
||||
fmt.Println("difference in sample sum")
|
||||
continue
|
||||
}
|
||||
if gh.SampleCount == nil || eh.SampleCount == nil || *(gh.SampleCount) != *(eh.SampleCount) {
|
||||
fmt.Println("difference in sample count")
|
||||
continue
|
||||
}
|
||||
|
||||
case got.Metric[i].Counter != nil && expected.Metric[j].Counter != nil:
|
||||
gc := got.Metric[i].Counter
|
||||
ec := expected.Metric[j].Counter
|
||||
if gc.Value == nil || *(gc.Value) != *(ec.Value) {
|
||||
fmt.Println("difference in counter values")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// this is a match
|
||||
matchCount = matchCount + 1
|
||||
break
|
||||
}
|
||||
gh := got.Metric[i].Histogram
|
||||
eh := expected.Metric[j].Histogram
|
||||
if gh == nil {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(gh.Bucket, eh.Bucket) {
|
||||
continue
|
||||
}
|
||||
// this is a sum record, expecting a latency which is more than the
|
||||
// expected one. If the sum is smaller than expected, it will be considered
|
||||
// as NOT a match
|
||||
if gh.SampleSum == nil || *(gh.SampleSum) < *(eh.SampleSum) {
|
||||
continue
|
||||
}
|
||||
if gh.SampleCount == nil || *(gh.SampleCount) != *(eh.SampleCount) {
|
||||
continue
|
||||
}
|
||||
// this is a match
|
||||
matchCount = matchCount + 1
|
||||
break
|
||||
}
|
||||
|
||||
if matchCount != numRecords {
|
||||
fmt.Printf("matchCount %v, numRecords %v\n", matchCount, numRecords)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return matchCount == numRecords
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user