Add generated file
This PR adds generated files under pkg/client and vendor folder.
This commit is contained in:
39
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/BUILD
generated
vendored
Normal file
39
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/BUILD
generated
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["etcd.go"],
|
||||
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/util/staticpod:go_default_library",
|
||||
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
|
||||
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["etcd_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/test:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
224
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/etcd.go
generated
vendored
Normal file
224
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/etcd.go
generated
vendored
Normal file
@@ -0,0 +1,224 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package etcd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/etcd/clientv3"
|
||||
"github.com/coreos/etcd/pkg/transport"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
|
||||
)
|
||||
|
||||
// ClusterInterrogator is an interface to get etcd cluster related information
|
||||
type ClusterInterrogator interface {
|
||||
ClusterAvailable() (bool, error)
|
||||
GetClusterStatus() (map[string]*clientv3.StatusResponse, error)
|
||||
GetClusterVersions() (map[string]string, error)
|
||||
GetVersion() (string, error)
|
||||
HasTLS() bool
|
||||
WaitForClusterAvailable(delay time.Duration, retries int, retryInterval time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// Client provides connection parameters for an etcd cluster
|
||||
type Client struct {
|
||||
Endpoints []string
|
||||
TLS *tls.Config
|
||||
}
|
||||
|
||||
// HasTLS returns true if etcd is configured for TLS
|
||||
func (c Client) HasTLS() bool {
|
||||
return c.TLS != nil
|
||||
}
|
||||
|
||||
// PodManifestsHaveTLS reads the etcd staticpod manifest from disk and returns false if the TLS flags
|
||||
// are missing from the command list. If all the flags are present it returns true.
|
||||
func PodManifestsHaveTLS(ManifestDir string) (bool, error) {
|
||||
etcdPodPath := constants.GetStaticPodFilepath(constants.Etcd, ManifestDir)
|
||||
etcdPod, err := staticpod.ReadStaticPodFromDisk(etcdPodPath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check if etcd pod implements TLS: %v", err)
|
||||
}
|
||||
|
||||
tlsFlags := []string{
|
||||
"--cert-file=",
|
||||
"--key-file=",
|
||||
"--trusted-ca-file=",
|
||||
"--client-cert-auth=",
|
||||
"--peer-cert-file=",
|
||||
"--peer-key-file=",
|
||||
"--peer-trusted-ca-file=",
|
||||
"--peer-client-cert-auth=",
|
||||
}
|
||||
FlagLoop:
|
||||
for _, flag := range tlsFlags {
|
||||
for _, container := range etcdPod.Spec.Containers {
|
||||
for _, arg := range container.Command {
|
||||
if strings.Contains(arg, flag) {
|
||||
continue FlagLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
// flag not found in any container
|
||||
return false, nil
|
||||
}
|
||||
// all flags were found in container args; pod fully implements TLS
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// New creates a new EtcdCluster client
|
||||
func New(endpoints []string, ca, cert, key string) (*Client, error) {
|
||||
client := Client{Endpoints: endpoints}
|
||||
|
||||
if ca != "" || cert != "" || key != "" {
|
||||
tlsInfo := transport.TLSInfo{
|
||||
CertFile: cert,
|
||||
KeyFile: key,
|
||||
TrustedCAFile: ca,
|
||||
}
|
||||
tlsConfig, err := tlsInfo.ClientConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.TLS = tlsConfig
|
||||
}
|
||||
|
||||
return &client, nil
|
||||
}
|
||||
|
||||
// NewFromStaticPod creates a GenericClient from the given endpoints, manifestDir, and certificatesDir
|
||||
func NewFromStaticPod(endpoints []string, manifestDir string, certificatesDir string) (*Client, error) {
|
||||
hasTLS, err := PodManifestsHaveTLS(manifestDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read manifests from: %s, error: %v", manifestDir, err)
|
||||
}
|
||||
if hasTLS {
|
||||
return New(
|
||||
endpoints,
|
||||
filepath.Join(certificatesDir, constants.EtcdCACertName),
|
||||
filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName),
|
||||
filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName),
|
||||
)
|
||||
}
|
||||
return New(endpoints, "", "", "")
|
||||
}
|
||||
|
||||
// GetVersion returns the etcd version of the cluster.
|
||||
// An error is returned if the version of all endpoints do not match
|
||||
func (c Client) GetVersion() (string, error) {
|
||||
var clusterVersion string
|
||||
|
||||
versions, err := c.GetClusterVersions()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, v := range versions {
|
||||
if clusterVersion != "" && clusterVersion != v {
|
||||
return "", fmt.Errorf("etcd cluster contains endpoints with mismatched versions: %v", versions)
|
||||
}
|
||||
clusterVersion = v
|
||||
}
|
||||
if clusterVersion == "" {
|
||||
return "", fmt.Errorf("could not determine cluster etcd version")
|
||||
}
|
||||
return clusterVersion, nil
|
||||
}
|
||||
|
||||
// GetClusterVersions returns a map of the endpoints and their associated versions
|
||||
func (c Client) GetClusterVersions() (map[string]string, error) {
|
||||
versions := make(map[string]string)
|
||||
statuses, err := c.GetClusterStatus()
|
||||
if err != nil {
|
||||
return versions, err
|
||||
}
|
||||
|
||||
for ep, status := range statuses {
|
||||
versions[ep] = status.Version
|
||||
}
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
// ClusterAvailable returns true if the cluster status indicates the cluster is available.
|
||||
func (c Client) ClusterAvailable() (bool, error) {
|
||||
_, err := c.GetClusterStatus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GetClusterStatus returns nil for status Up or error for status Down
|
||||
func (c Client) GetClusterStatus() (map[string]*clientv3.StatusResponse, error) {
|
||||
cli, err := clientv3.New(clientv3.Config{
|
||||
Endpoints: c.Endpoints,
|
||||
DialTimeout: 5 * time.Second,
|
||||
TLS: c.TLS,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
clusterStatus := make(map[string]*clientv3.StatusResponse)
|
||||
for _, ep := range c.Endpoints {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
resp, err := cli.Status(ctx, ep)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clusterStatus[ep] = resp
|
||||
}
|
||||
return clusterStatus, nil
|
||||
}
|
||||
|
||||
// WaitForClusterAvailable returns true if all endpoints in the cluster are available after an initial delay and retry attempts, an error is returned otherwise
|
||||
func (c Client) WaitForClusterAvailable(delay time.Duration, retries int, retryInterval time.Duration) (bool, error) {
|
||||
fmt.Printf("[util/etcd] Waiting %v for initial delay\n", delay)
|
||||
time.Sleep(delay)
|
||||
for i := 0; i < retries; i++ {
|
||||
if i > 0 {
|
||||
fmt.Printf("[util/etcd] Waiting %v until next retry\n", retryInterval)
|
||||
time.Sleep(retryInterval)
|
||||
}
|
||||
fmt.Printf("[util/etcd] Attempting to see if all cluster endpoints are available %d/%d\n", i+1, retries)
|
||||
resp, err := c.ClusterAvailable()
|
||||
if err != nil {
|
||||
switch err {
|
||||
case context.DeadlineExceeded:
|
||||
fmt.Println("[util/etcd] Attempt timed out")
|
||||
default:
|
||||
fmt.Printf("[util/etcd] Attempt failed with error: %v\n", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return false, fmt.Errorf("timeout waiting for etcd cluster to be available")
|
||||
}
|
||||
|
||||
// CheckConfigurationIsHA returns true if the given MasterConfiguration etcd block appears to be an HA configuration.
|
||||
func CheckConfigurationIsHA(cfg *kubeadmapi.Etcd) bool {
|
||||
return cfg.External != nil && len(cfg.External.Endpoints) > 1
|
||||
}
|
310
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/etcd_test.go
generated
vendored
Normal file
310
vendor/k8s.io/kubernetes/cmd/kubeadm/app/util/etcd/etcd_test.go
generated
vendored
Normal file
@@ -0,0 +1,310 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package etcd
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
|
||||
)
|
||||
|
||||
const (
|
||||
secureEtcdPod = `# generated by kubeadm v1.10.0
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: etcd
|
||||
tier: control-plane
|
||||
name: etcd
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- etcd
|
||||
- --advertise-client-urls=https://127.0.0.1:2379
|
||||
- --data-dir=/var/lib/etcd
|
||||
- --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
|
||||
- --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
|
||||
- --listen-client-urls=https://127.0.0.1:2379
|
||||
- --peer-client-cert-auth=true
|
||||
- --cert-file=/etc/kubernetes/pki/etcd/server.crt
|
||||
- --key-file=/etc/kubernetes/pki/etcd/server.key
|
||||
- --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
|
||||
- --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
|
||||
- --client-cert-auth=true
|
||||
image: k8s.gcr.io/etcd-amd64:3.1.12
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- ETCDCTL_API=3 etcdctl --endpoints=127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt
|
||||
--cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
|
||||
get foo
|
||||
failureThreshold: 8
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: etcd
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/etcd
|
||||
name: etcd-data
|
||||
- mountPath: /etc/kubernetes/pki/etcd
|
||||
name: etcd-certs
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/etcd
|
||||
type: DirectoryOrCreate
|
||||
name: etcd-data
|
||||
- hostPath:
|
||||
path: /etc/kubernetes/pki/etcd
|
||||
type: DirectoryOrCreate
|
||||
name: etcd-certs
|
||||
status: {}
|
||||
`
|
||||
secureExposedEtcdPod = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: etcd
|
||||
tier: control-plane
|
||||
name: etcd
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- etcd
|
||||
- --advertise-client-urls=https://10.0.5.5:2379
|
||||
- --data-dir=/var/lib/etcd
|
||||
- --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
|
||||
- --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
|
||||
- --listen-client-urls=https://[::0:0]:2379
|
||||
- --peer-client-cert-auth=true
|
||||
- --cert-file=/etc/kubernetes/pki/etcd/server.crt
|
||||
- --key-file=/etc/kubernetes/pki/etcd/server.key
|
||||
- --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
|
||||
- --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
|
||||
- --client-cert-auth=true
|
||||
image: k8s.gcr.io/etcd-amd64:3.1.12
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- ETCDCTL_API=3 etcdctl --endpoints=https://[::1]:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt
|
||||
--cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
|
||||
get foo
|
||||
failureThreshold: 8
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: etcd
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/etcd
|
||||
name: etcd-data
|
||||
- mountPath: /etc/kubernetes/pki/etcd
|
||||
name: etcd-certs
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/etcd
|
||||
type: DirectoryOrCreate
|
||||
name: etcd-data
|
||||
- hostPath:
|
||||
path: /etc/kubernetes/pki/etcd
|
||||
type: DirectoryOrCreate
|
||||
name: etcd-certs
|
||||
status: {}
|
||||
`
|
||||
insecureEtcdPod = `# generated by kubeadm v1.9.6
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: etcd
|
||||
tier: control-plane
|
||||
name: etcd
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- etcd
|
||||
- --listen-client-urls=http://127.0.0.1:2379
|
||||
- --advertise-client-urls=http://127.0.0.1:2379
|
||||
- --data-dir=/var/lib/etcd
|
||||
image: gcr.io/google_containers/etcd-amd64:3.1.11
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /health
|
||||
port: 2379
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: etcd
|
||||
resources: {}
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/etcd
|
||||
name: etcd
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/etcd
|
||||
type: DirectoryOrCreate
|
||||
name: etcd
|
||||
status: {}
|
||||
`
|
||||
invalidPod = `---{ broken yaml @@@`
|
||||
)
|
||||
|
||||
func TestPodManifestHasTLS(t *testing.T) {
|
||||
tests := []struct {
|
||||
description string
|
||||
podYaml string
|
||||
hasTLS bool
|
||||
expectErr bool
|
||||
writeManifest bool
|
||||
}{
|
||||
{
|
||||
description: "secure etcd returns true",
|
||||
podYaml: secureEtcdPod,
|
||||
hasTLS: true,
|
||||
writeManifest: true,
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
description: "secure exposed etcd returns true",
|
||||
podYaml: secureExposedEtcdPod,
|
||||
hasTLS: true,
|
||||
writeManifest: true,
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
description: "insecure etcd returns false",
|
||||
podYaml: insecureEtcdPod,
|
||||
hasTLS: false,
|
||||
writeManifest: true,
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
description: "invalid pod fails to unmarshal",
|
||||
podYaml: invalidPod,
|
||||
hasTLS: false,
|
||||
writeManifest: true,
|
||||
expectErr: true,
|
||||
},
|
||||
{
|
||||
description: "non-existent file returns error",
|
||||
podYaml: ``,
|
||||
hasTLS: false,
|
||||
writeManifest: false,
|
||||
expectErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
tmpdir := testutil.SetupTempDir(t)
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
manifestPath := filepath.Join(tmpdir, "etcd.yaml")
|
||||
if rt.writeManifest {
|
||||
err := ioutil.WriteFile(manifestPath, []byte(rt.podYaml), 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to write pod manifest\n%s\n\tfatal error: %v", rt.description, err)
|
||||
}
|
||||
}
|
||||
|
||||
hasTLS, actualErr := PodManifestsHaveTLS(tmpdir)
|
||||
if (actualErr != nil) != rt.expectErr {
|
||||
t.Errorf(
|
||||
"PodManifestHasTLS failed\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v",
|
||||
rt.description,
|
||||
rt.expectErr,
|
||||
(actualErr != nil),
|
||||
actualErr,
|
||||
)
|
||||
}
|
||||
|
||||
if hasTLS != rt.hasTLS {
|
||||
t.Errorf("PodManifestHasTLS failed\n%s\n\texpected hasTLS: %t\n\tgot: %t", rt.description, rt.hasTLS, hasTLS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckConfigurationIsHA(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
cfg *kubeadmapi.Etcd
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "HA etcd",
|
||||
cfg: &kubeadmapi.Etcd{
|
||||
External: &kubeadmapi.ExternalEtcd{
|
||||
Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"},
|
||||
},
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "single External etcd",
|
||||
cfg: &kubeadmapi.Etcd{
|
||||
External: &kubeadmapi.ExternalEtcd{
|
||||
Endpoints: []string{"10.100.0.1:2379"},
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "local etcd",
|
||||
cfg: &kubeadmapi.Etcd{
|
||||
Local: &kubeadmapi.LocalEtcd{},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "empty etcd struct",
|
||||
cfg: &kubeadmapi.Etcd{},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if isHA := CheckConfigurationIsHA(test.cfg); isHA != test.expected {
|
||||
t.Errorf("expected isHA to be %v, got %v", test.expected, isHA)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user