Add generated file

This PR adds generated files under pkg/client and vendor folder.
This commit is contained in:
xing-yang
2018-07-12 10:55:15 -07:00
parent 36b1de0341
commit e213d1890d
17729 changed files with 5090889 additions and 0 deletions

View File

@@ -0,0 +1,103 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"metadata.go",
"openstack.go",
"openstack_client.go",
"openstack_instances.go",
"openstack_loadbalancer.go",
"openstack_metrics.go",
"openstack_routes.go",
"openstack_volumes.go",
],
importpath = "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack",
deps = [
"//pkg/api/v1/service:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/gophercloud/gophercloud:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/tokens:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/networks:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
"//vendor/github.com/mitchellh/mapstructure:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"metadata_test.go",
"openstack_routes_test.go",
"openstack_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/cloudprovider:go_default_library",
"//vendor/github.com/gophercloud/gophercloud:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,6 @@
# Maintainers
* [Angus Lees](https://github.com/anguslees)
[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/pkg/cloudprovider/providers/openstack/MAINTAINERS.md?pixel)]()

View File

@@ -0,0 +1,10 @@
approvers:
- anguslees
- NickrenREN
- dims
- FengyunPan2
reviewers:
- anguslees
- NickrenREN
- dims
- FengyunPan2

View File

@@ -0,0 +1,198 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/utils/exec"
)
const (
// metadataURLTemplate allows building an OpenStack Metadata service URL.
// It's a hardcoded IPv4 link-local address as documented in "OpenStack Cloud
// Administrator Guide", chapter Compute - Networking with nova-network.
//https://docs.openstack.org/nova/latest/admin/networking-nova.html#metadata-service
defaultMetadataVersion = "2012-08-10"
metadataURLTemplate = "http://169.254.169.254/openstack/%s/meta_data.json"
// metadataID is used as an identifier on the metadata search order configuration.
metadataID = "metadataService"
// Config drive is defined as an iso9660 or vfat (deprecated) drive
// with the "config-2" label.
//https://docs.openstack.org/nova/latest/user/config-drive.html
configDriveLabel = "config-2"
configDrivePathTemplate = "openstack/%s/meta_data.json"
// configDriveID is used as an identifier on the metadata search order configuration.
configDriveID = "configDrive"
)
// ErrBadMetadata is used to indicate a problem parsing data from metadata server
var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid")
// DeviceMetadata is a single/simplified data structure for all kinds of device metadata types.
type DeviceMetadata struct {
Type string `json:"type"`
Bus string `json:"bus,omitempty"`
Serial string `json:"serial,omitempty"`
Address string `json:"address,omitempty"`
// .. and other fields.
}
// Metadata has the information fetched from OpenStack metadata service or
// config drives. Assumes the "2012-08-10" meta_data.json format.
// See http://docs.openstack.org/user-guide/cli_config_drive.html
type Metadata struct {
UUID string `json:"uuid"`
Name string `json:"name"`
AvailabilityZone string `json:"availability_zone"`
Devices []DeviceMetadata `json:"devices,omitempty"`
// .. and other fields we don't care about. Expand as necessary.
}
// parseMetadata reads JSON from OpenStack metadata server and parses
// instance ID out of it.
func parseMetadata(r io.Reader) (*Metadata, error) {
var metadata Metadata
json := json.NewDecoder(r)
if err := json.Decode(&metadata); err != nil {
return nil, err
}
if metadata.UUID == "" {
return nil, ErrBadMetadata
}
return &metadata, nil
}
func getMetadataURL(metadataVersion string) string {
return fmt.Sprintf(metadataURLTemplate, metadataVersion)
}
func getConfigDrivePath(metadataVersion string) string {
return fmt.Sprintf(configDrivePathTemplate, metadataVersion)
}
func getMetadataFromConfigDrive(metadataVersion string) (*Metadata, error) {
// Try to read instance UUID from config drive.
dev := "/dev/disk/by-label/" + configDriveLabel
if _, err := os.Stat(dev); os.IsNotExist(err) {
out, err := exec.New().Command(
"blkid", "-l",
"-t", "LABEL="+configDriveLabel,
"-o", "device",
).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("unable to run blkid: %v", err)
}
dev = strings.TrimSpace(string(out))
}
mntdir, err := ioutil.TempDir("", "configdrive")
if err != nil {
return nil, err
}
defer os.Remove(mntdir)
glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)
mounter := mount.New("" /* default mount path */)
err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
if err != nil {
err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})
}
if err != nil {
return nil, fmt.Errorf("error mounting configdrive %s: %v", dev, err)
}
defer mounter.Unmount(mntdir)
glog.V(4).Infof("Configdrive mounted on %s", mntdir)
configDrivePath := getConfigDrivePath(metadataVersion)
f, err := os.Open(
filepath.Join(mntdir, configDrivePath))
if err != nil {
return nil, fmt.Errorf("error reading %s on config drive: %v", configDrivePath, err)
}
defer f.Close()
return parseMetadata(f)
}
func getMetadataFromMetadataService(metadataVersion string) (*Metadata, error) {
// Try to get JSON from metadata server.
metadataURL := getMetadataURL(metadataVersion)
glog.V(4).Infof("Attempting to fetch metadata from %s", metadataURL)
resp, err := http.Get(metadataURL)
if err != nil {
return nil, fmt.Errorf("error fetching %s: %v", metadataURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("unexpected status code when reading metadata from %s: %s", metadataURL, resp.Status)
return nil, err
}
return parseMetadata(resp.Body)
}
// Metadata is fixed for the current host, so cache the value process-wide
var metadataCache *Metadata
func getMetadata(order string) (*Metadata, error) {
if metadataCache == nil {
var md *Metadata
var err error
elements := strings.Split(order, ",")
for _, id := range elements {
id = strings.TrimSpace(id)
switch id {
case configDriveID:
md, err = getMetadataFromConfigDrive(defaultMetadataVersion)
case metadataID:
md, err = getMetadataFromMetadataService(defaultMetadataVersion)
default:
err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, configDriveID, metadataID)
}
if err == nil {
break
}
}
if err != nil {
return nil, err
}
metadataCache = md
}
return metadataCache, nil
}

View File

@@ -0,0 +1,115 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"strings"
"testing"
)
var FakeMetadata = Metadata{
UUID: "83679162-1378-4288-a2d4-70e13ec132aa",
Name: "test",
AvailabilityZone: "nova",
}
func SetMetadataFixture(value *Metadata) {
metadataCache = value
}
func ClearMetadata() {
metadataCache = nil
}
func TestParseMetadata(t *testing.T) {
_, err := parseMetadata(strings.NewReader("bogus"))
if err == nil {
t.Errorf("Should fail when bad data is provided: %s", err)
}
data := strings.NewReader(`
{
"availability_zone": "nova",
"files": [
{
"content_path": "/content/0000",
"path": "/etc/network/interfaces"
},
{
"content_path": "/content/0001",
"path": "known_hosts"
}
],
"hostname": "test.novalocal",
"launch_index": 0,
"name": "test",
"meta": {
"role": "webservers",
"essential": "false"
},
"public_keys": {
"mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n"
},
"uuid": "83679162-1378-4288-a2d4-70e13ec132aa",
"devices": [
{
"bus": "scsi",
"serial": "6df1888b-f373-41cf-b960-3786e60a28ef",
"tags": ["fake_tag"],
"type": "disk",
"address": "0:0:0:0"
}
]
}
`)
md, err := parseMetadata(data)
if err != nil {
t.Fatalf("Should succeed when provided with valid data: %s", err)
}
if md.Name != "test" {
t.Errorf("incorrect name: %s", md.Name)
}
if md.UUID != "83679162-1378-4288-a2d4-70e13ec132aa" {
t.Errorf("incorrect uuid: %s", md.UUID)
}
if md.AvailabilityZone != "nova" {
t.Errorf("incorrect az: %s", md.AvailabilityZone)
}
if len(md.Devices) != 1 {
t.Errorf("expecting to find 1 device, found %d", len(md.Devices))
}
if md.Devices[0].Bus != "scsi" {
t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus)
}
if md.Devices[0].Address != "0:0:0:0" {
t.Errorf("incorrect disk address: %s", md.Devices[0].Address)
}
if md.Devices[0].Type != "disk" {
t.Errorf("incorrect device type: %s", md.Devices[0].Type)
}
if md.Devices[0].Serial != "6df1888b-f373-41cf-b960-3786e60a28ef" {
t.Errorf("incorrect device serial: %s", md.Devices[0].Serial)
}
}

View File

@@ -0,0 +1,807 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"regexp"
"strings"
"time"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack"
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
"github.com/gophercloud/gophercloud/pagination"
"github.com/mitchellh/mapstructure"
"gopkg.in/gcfg.v1"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
netutil "k8s.io/apimachinery/pkg/util/net"
certutil "k8s.io/client-go/util/cert"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/controller"
)
const (
// ProviderName is the name of the openstack provider
ProviderName = "openstack"
availabilityZone = "availability_zone"
defaultTimeOut = 60 * time.Second
)
// ErrNotFound is used to inform that the object is missing
var ErrNotFound = errors.New("failed to find object")
// ErrMultipleResults is used when we unexpectedly get back multiple results
var ErrMultipleResults = errors.New("multiple results where only one expected")
// ErrNoAddressFound is used when we cannot find an ip address for the host
var ErrNoAddressFound = errors.New("no address found for host")
// MyDuration is the encoding.TextUnmarshaler interface for time.Duration
type MyDuration struct {
time.Duration
}
// UnmarshalText is used to convert from text to Duration
func (d *MyDuration) UnmarshalText(text []byte) error {
res, err := time.ParseDuration(string(text))
if err != nil {
return err
}
d.Duration = res
return nil
}
// LoadBalancer is used for creating and maintaining load balancers
type LoadBalancer struct {
network *gophercloud.ServiceClient
compute *gophercloud.ServiceClient
lb *gophercloud.ServiceClient
opts LoadBalancerOpts
}
// LoadBalancerOpts have the options to talk to Neutron LBaaSV2 or Octavia
type LoadBalancerOpts struct {
LBVersion string `gcfg:"lb-version"` // overrides autodetection. Only support v2.
UseOctavia bool `gcfg:"use-octavia"` // uses Octavia V2 service catalog endpoint
SubnetID string `gcfg:"subnet-id"` // overrides autodetection.
FloatingNetworkID string `gcfg:"floating-network-id"` // If specified, will create floating ip for loadbalancer, or do not create floating ip.
LBMethod string `gcfg:"lb-method"` // default to ROUND_ROBIN.
LBProvider string `gcfg:"lb-provider"`
CreateMonitor bool `gcfg:"create-monitor"`
MonitorDelay MyDuration `gcfg:"monitor-delay"`
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
ManageSecurityGroups bool `gcfg:"manage-security-groups"`
NodeSecurityGroupIDs []string // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache
}
// BlockStorageOpts is used to talk to Cinder service
type BlockStorageOpts struct {
BSVersion string `gcfg:"bs-version"` // overrides autodetection. v1 or v2. Defaults to auto
TrustDevicePath bool `gcfg:"trust-device-path"` // See Issue #33128
IgnoreVolumeAZ bool `gcfg:"ignore-volume-az"`
}
// RouterOpts is used for Neutron routes
type RouterOpts struct {
RouterID string `gcfg:"router-id"` // required
}
// MetadataOpts is used for configuring how to talk to metadata service or config drive
type MetadataOpts struct {
SearchOrder string `gcfg:"search-order"`
RequestTimeout MyDuration `gcfg:"request-timeout"`
}
// OpenStack is an implementation of cloud provider Interface for OpenStack.
type OpenStack struct {
provider *gophercloud.ProviderClient
region string
lbOpts LoadBalancerOpts
bsOpts BlockStorageOpts
routeOpts RouterOpts
metadataOpts MetadataOpts
// InstanceID of the server where this OpenStack object is instantiated.
localInstanceID string
}
// Config is used to read and store information from the cloud configuration file
type Config struct {
Global struct {
AuthURL string `gcfg:"auth-url"`
Username string
UserID string `gcfg:"user-id"`
Password string
TenantID string `gcfg:"tenant-id"`
TenantName string `gcfg:"tenant-name"`
TrustID string `gcfg:"trust-id"`
DomainID string `gcfg:"domain-id"`
DomainName string `gcfg:"domain-name"`
Region string
CAFile string `gcfg:"ca-file"`
}
LoadBalancer LoadBalancerOpts
BlockStorage BlockStorageOpts
Route RouterOpts
Metadata MetadataOpts
}
func init() {
registerMetrics()
cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
cfg, err := readConfig(config)
if err != nil {
return nil, err
}
return newOpenStack(cfg)
})
}
func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
return gophercloud.AuthOptions{
IdentityEndpoint: cfg.Global.AuthURL,
Username: cfg.Global.Username,
UserID: cfg.Global.UserID,
Password: cfg.Global.Password,
TenantID: cfg.Global.TenantID,
TenantName: cfg.Global.TenantName,
DomainID: cfg.Global.DomainID,
DomainName: cfg.Global.DomainName,
// Persistent service, so we need to be able to renew tokens.
AllowReauth: true,
}
}
func (cfg Config) toAuth3Options() tokens3.AuthOptions {
return tokens3.AuthOptions{
IdentityEndpoint: cfg.Global.AuthURL,
Username: cfg.Global.Username,
UserID: cfg.Global.UserID,
Password: cfg.Global.Password,
DomainID: cfg.Global.DomainID,
DomainName: cfg.Global.DomainName,
AllowReauth: true,
}
}
// configFromEnv allows setting up credentials etc using the
// standard OS_* OpenStack client environment variables.
func configFromEnv() (cfg Config, ok bool) {
cfg.Global.AuthURL = os.Getenv("OS_AUTH_URL")
cfg.Global.Username = os.Getenv("OS_USERNAME")
cfg.Global.Password = os.Getenv("OS_PASSWORD")
cfg.Global.Region = os.Getenv("OS_REGION_NAME")
cfg.Global.UserID = os.Getenv("OS_USER_ID")
cfg.Global.TrustID = os.Getenv("OS_TRUST_ID")
cfg.Global.TenantID = os.Getenv("OS_TENANT_ID")
if cfg.Global.TenantID == "" {
cfg.Global.TenantID = os.Getenv("OS_PROJECT_ID")
}
cfg.Global.TenantName = os.Getenv("OS_TENANT_NAME")
if cfg.Global.TenantName == "" {
cfg.Global.TenantName = os.Getenv("OS_PROJECT_NAME")
}
cfg.Global.DomainID = os.Getenv("OS_DOMAIN_ID")
if cfg.Global.DomainID == "" {
cfg.Global.DomainID = os.Getenv("OS_USER_DOMAIN_ID")
}
cfg.Global.DomainName = os.Getenv("OS_DOMAIN_NAME")
if cfg.Global.DomainName == "" {
cfg.Global.DomainName = os.Getenv("OS_USER_DOMAIN_NAME")
}
ok = cfg.Global.AuthURL != "" &&
cfg.Global.Username != "" &&
cfg.Global.Password != "" &&
(cfg.Global.TenantID != "" || cfg.Global.TenantName != "" ||
cfg.Global.DomainID != "" || cfg.Global.DomainName != "" ||
cfg.Global.Region != "" || cfg.Global.UserID != "" ||
cfg.Global.TrustID != "")
cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", configDriveID, metadataID)
cfg.BlockStorage.BSVersion = "auto"
return
}
func readConfig(config io.Reader) (Config, error) {
if config == nil {
return Config{}, fmt.Errorf("no OpenStack cloud provider config file given")
}
cfg, _ := configFromEnv()
// Set default values for config params
cfg.BlockStorage.BSVersion = "auto"
cfg.BlockStorage.TrustDevicePath = false
cfg.BlockStorage.IgnoreVolumeAZ = false
cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", configDriveID, metadataID)
err := gcfg.ReadInto(&cfg, config)
return cfg, err
}
// caller is a tiny helper for conditional unwind logic
type caller bool
func newCaller() caller { return caller(true) }
func (c *caller) disarm() { *c = false }
func (c *caller) call(f func()) {
if *c {
f()
}
}
func readInstanceID(searchOrder string) (string, error) {
// Try to find instance ID on the local filesystem (created by cloud-init)
const instanceIDFile = "/var/lib/cloud/data/instance-id"
idBytes, err := ioutil.ReadFile(instanceIDFile)
if err == nil {
instanceID := string(idBytes)
instanceID = strings.TrimSpace(instanceID)
glog.V(3).Infof("Got instance id from %s: %s", instanceIDFile, instanceID)
if instanceID != "" {
return instanceID, nil
}
// Fall through to metadata server lookup
}
md, err := getMetadata(searchOrder)
if err != nil {
return "", err
}
return md.UUID, nil
}
// check opts for OpenStack
func checkOpenStackOpts(openstackOpts *OpenStack) error {
lbOpts := openstackOpts.lbOpts
// if need to create health monitor for Neutron LB,
// monitor-delay, monitor-timeout and monitor-max-retries should be set.
emptyDuration := MyDuration{}
if lbOpts.CreateMonitor {
if lbOpts.MonitorDelay == emptyDuration {
return fmt.Errorf("monitor-delay not set in cloud provider config")
}
if lbOpts.MonitorTimeout == emptyDuration {
return fmt.Errorf("monitor-timeout not set in cloud provider config")
}
if lbOpts.MonitorMaxRetries == uint(0) {
return fmt.Errorf("monitor-max-retries not set in cloud provider config")
}
}
return checkMetadataSearchOrder(openstackOpts.metadataOpts.SearchOrder)
}
func newOpenStack(cfg Config) (*OpenStack, error) {
provider, err := openstack.NewClient(cfg.Global.AuthURL)
if err != nil {
return nil, err
}
if cfg.Global.CAFile != "" {
roots, err := certutil.NewPool(cfg.Global.CAFile)
if err != nil {
return nil, err
}
config := &tls.Config{}
config.RootCAs = roots
provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})
}
if cfg.Global.TrustID != "" {
opts := cfg.toAuth3Options()
authOptsExt := trusts.AuthOptsExt{
TrustID: cfg.Global.TrustID,
AuthOptionsBuilder: &opts,
}
err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{})
} else {
err = openstack.Authenticate(provider, cfg.toAuthOptions())
}
if err != nil {
return nil, err
}
emptyDuration := MyDuration{}
if cfg.Metadata.RequestTimeout == emptyDuration {
cfg.Metadata.RequestTimeout.Duration = time.Duration(defaultTimeOut)
}
provider.HTTPClient.Timeout = cfg.Metadata.RequestTimeout.Duration
os := OpenStack{
provider: provider,
region: cfg.Global.Region,
lbOpts: cfg.LoadBalancer,
bsOpts: cfg.BlockStorage,
routeOpts: cfg.Route,
metadataOpts: cfg.Metadata,
}
err = checkOpenStackOpts(&os)
if err != nil {
return nil, err
}
return &os, nil
}
// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
func (os *OpenStack) Initialize(clientBuilder controller.ControllerClientBuilder) {}
// mapNodeNameToServerName maps a k8s NodeName to an OpenStack Server Name
// This is a simple string cast.
func mapNodeNameToServerName(nodeName types.NodeName) string {
return string(nodeName)
}
// GetNodeNameByID maps instanceid to types.NodeName
func (os *OpenStack) GetNodeNameByID(instanceID string) (types.NodeName, error) {
client, err := os.NewComputeV2()
var nodeName types.NodeName
if err != nil {
return nodeName, err
}
server, err := servers.Get(client, instanceID).Extract()
if err != nil {
return nodeName, err
}
nodeName = mapServerToNodeName(server)
return nodeName, nil
}
// mapServerToNodeName maps an OpenStack Server to a k8s NodeName
func mapServerToNodeName(server *servers.Server) types.NodeName {
// Node names are always lowercase, and (at least)
// routecontroller does case-sensitive string comparisons
// assuming this
return types.NodeName(strings.ToLower(server.Name))
}
func foreachServer(client *gophercloud.ServiceClient, opts servers.ListOptsBuilder, handler func(*servers.Server) (bool, error)) error {
pager := servers.List(client, opts)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
s, err := servers.ExtractServers(page)
if err != nil {
return false, err
}
for _, server := range s {
ok, err := handler(&server)
if !ok || err != nil {
return false, err
}
}
return true, nil
})
return err
}
func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*servers.Server, error) {
opts := servers.ListOpts{
Name: fmt.Sprintf("^%s$", regexp.QuoteMeta(mapNodeNameToServerName(name))),
}
pager := servers.List(client, opts)
serverList := make([]servers.Server, 0, 1)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
s, err := servers.ExtractServers(page)
if err != nil {
return false, err
}
serverList = append(serverList, s...)
if len(serverList) > 1 {
return false, ErrMultipleResults
}
return true, nil
})
if err != nil {
return nil, err
}
if len(serverList) == 0 {
return nil, ErrNotFound
}
return &serverList[0], nil
}
func nodeAddresses(srv *servers.Server) ([]v1.NodeAddress, error) {
addrs := []v1.NodeAddress{}
type Address struct {
IPType string `mapstructure:"OS-EXT-IPS:type"`
Addr string
}
var addresses map[string][]Address
err := mapstructure.Decode(srv.Addresses, &addresses)
if err != nil {
return nil, err
}
for network, addrList := range addresses {
for _, props := range addrList {
var addressType v1.NodeAddressType
if props.IPType == "floating" || network == "public" {
addressType = v1.NodeExternalIP
} else {
addressType = v1.NodeInternalIP
}
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: addressType,
Address: props.Addr,
},
)
}
}
// AccessIPs are usually duplicates of "public" addresses.
if srv.AccessIPv4 != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv4,
},
)
}
if srv.AccessIPv6 != "" {
v1helper.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: srv.AccessIPv6,
},
)
}
return addrs, nil
}
func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName) ([]v1.NodeAddress, error) {
srv, err := getServerByName(client, name)
if err != nil {
return nil, err
}
return nodeAddresses(srv)
}
func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName, needIPv6 bool) (string, error) {
addrs, err := getAddressesByName(client, name)
if err != nil {
return "", err
} else if len(addrs) == 0 {
return "", ErrNoAddressFound
}
for _, addr := range addrs {
isIPv6 := net.ParseIP(addr.Address).To4() == nil
if (addr.Type == v1.NodeInternalIP) && (isIPv6 == needIPv6) {
return addr.Address, nil
}
}
for _, addr := range addrs {
isIPv6 := net.ParseIP(addr.Address).To4() == nil
if (addr.Type == v1.NodeExternalIP) && (isIPv6 == needIPv6) {
return addr.Address, nil
}
}
// It should never return an address from a different IP Address family than the one needed
return "", ErrNoAddressFound
}
// getAttachedInterfacesByID returns the node interfaces of the specified instance.
func getAttachedInterfacesByID(client *gophercloud.ServiceClient, serviceID string) ([]attachinterfaces.Interface, error) {
var interfaces []attachinterfaces.Interface
pager := attachinterfaces.List(client, serviceID)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
s, err := attachinterfaces.ExtractInterfaces(page)
if err != nil {
return false, err
}
interfaces = append(interfaces, s...)
return true, nil
})
if err != nil {
return interfaces, err
}
return interfaces, nil
}
// Clusters is a no-op
func (os *OpenStack) Clusters() (cloudprovider.Clusters, bool) {
return nil, false
}
// ProviderName returns the cloud provider ID.
func (os *OpenStack) ProviderName() string {
return ProviderName
}
// HasClusterID returns true if the cluster has a clusterID
func (os *OpenStack) HasClusterID() bool {
return true
}
// LoadBalancer initializes a LbaasV2 object
func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
glog.V(4).Info("openstack.LoadBalancer() called")
network, err := os.NewNetworkV2()
if err != nil {
return nil, false
}
compute, err := os.NewComputeV2()
if err != nil {
return nil, false
}
lb, err := os.NewLoadBalancerV2()
if err != nil {
return nil, false
}
// LBaaS v1 is deprecated in the OpenStack Liberty release.
// Currently kubernetes OpenStack cloud provider just support LBaaS v2.
lbVersion := os.lbOpts.LBVersion
if lbVersion != "" && lbVersion != "v2" {
glog.Warningf("Config error: currently only support LBaaS v2, unrecognised lb-version \"%v\"", lbVersion)
return nil, false
}
glog.V(1).Info("Claiming to support LoadBalancer")
return &LbaasV2{LoadBalancer{network, compute, lb, os.lbOpts}}, true
}
func isNotFound(err error) bool {
if _, ok := err.(gophercloud.ErrDefault404); ok {
return true
}
if errCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok {
if errCode.Actual == http.StatusNotFound {
return true
}
}
return false
}
// Zones indicates that we support zones
func (os *OpenStack) Zones() (cloudprovider.Zones, bool) {
glog.V(1).Info("Claiming to support Zones")
return os, true
}
// GetZone returns the current zone
func (os *OpenStack) GetZone(ctx context.Context) (cloudprovider.Zone, error) {
md, err := getMetadata(os.metadataOpts.SearchOrder)
if err != nil {
return cloudprovider.Zone{}, err
}
zone := cloudprovider.Zone{
FailureDomain: md.AvailabilityZone,
Region: os.region,
}
glog.V(4).Infof("Current zone is %v", zone)
return zone, nil
}
// GetZoneByProviderID implements Zones.GetZoneByProviderID
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
func (os *OpenStack) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) {
instanceID, err := instanceIDFromProviderID(providerID)
if err != nil {
return cloudprovider.Zone{}, err
}
compute, err := os.NewComputeV2()
if err != nil {
return cloudprovider.Zone{}, err
}
srv, err := servers.Get(compute, instanceID).Extract()
if err != nil {
return cloudprovider.Zone{}, err
}
zone := cloudprovider.Zone{
FailureDomain: srv.Metadata[availabilityZone],
Region: os.region,
}
glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone)
return zone, nil
}
// GetZoneByNodeName implements Zones.GetZoneByNodeName
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
func (os *OpenStack) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (cloudprovider.Zone, error) {
compute, err := os.NewComputeV2()
if err != nil {
return cloudprovider.Zone{}, err
}
srv, err := getServerByName(compute, nodeName)
if err != nil {
if err == ErrNotFound {
return cloudprovider.Zone{}, cloudprovider.InstanceNotFound
}
return cloudprovider.Zone{}, err
}
zone := cloudprovider.Zone{
FailureDomain: srv.Metadata[availabilityZone],
Region: os.region,
}
glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone)
return zone, nil
}
// Routes initializes routes support
func (os *OpenStack) Routes() (cloudprovider.Routes, bool) {
glog.V(4).Info("openstack.Routes() called")
network, err := os.NewNetworkV2()
if err != nil {
return nil, false
}
netExts, err := networkExtensions(network)
if err != nil {
glog.Warningf("Failed to list neutron extensions: %v", err)
return nil, false
}
if !netExts["extraroute"] {
glog.V(3).Info("Neutron extraroute extension not found, required for Routes support")
return nil, false
}
compute, err := os.NewComputeV2()
if err != nil {
return nil, false
}
r, err := NewRoutes(compute, network, os.routeOpts)
if err != nil {
glog.Warningf("Error initialising Routes support: %v", err)
return nil, false
}
glog.V(1).Info("Claiming to support Routes")
return r, true
}
func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) {
bsVersion := ""
if forceVersion == "" {
bsVersion = os.bsOpts.BSVersion
} else {
bsVersion = forceVersion
}
switch bsVersion {
case "v1":
sClient, err := os.NewBlockStorageV1()
if err != nil {
return nil, err
}
glog.V(3).Info("Using Blockstorage API V1")
return &VolumesV1{sClient, os.bsOpts}, nil
case "v2":
sClient, err := os.NewBlockStorageV2()
if err != nil {
return nil, err
}
glog.V(3).Info("Using Blockstorage API V2")
return &VolumesV2{sClient, os.bsOpts}, nil
case "v3":
sClient, err := os.NewBlockStorageV3()
if err != nil {
return nil, err
}
glog.V(3).Info("Using Blockstorage API V3")
return &VolumesV3{sClient, os.bsOpts}, nil
case "auto":
// Currently kubernetes support Cinder v1 / Cinder v2 / Cinder v3.
// Choose Cinder v3 firstly, if kubernetes can't initialize cinder v3 client, try to initialize cinder v2 client.
// If kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client.
// Return appropriate message when kubernetes can't initialize them.
if sClient, err := os.NewBlockStorageV3(); err == nil {
glog.V(3).Info("Using Blockstorage API V3")
return &VolumesV3{sClient, os.bsOpts}, nil
}
if sClient, err := os.NewBlockStorageV2(); err == nil {
glog.V(3).Info("Using Blockstorage API V2")
return &VolumesV2{sClient, os.bsOpts}, nil
}
if sClient, err := os.NewBlockStorageV1(); err == nil {
glog.V(3).Info("Using Blockstorage API V1")
return &VolumesV1{sClient, os.bsOpts}, nil
}
errTxt := "BlockStorage API version autodetection failed. " +
"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
return nil, errors.New(errTxt)
default:
errTxt := fmt.Sprintf("Config error: unrecognised bs-version \"%v\"", os.bsOpts.BSVersion)
return nil, errors.New(errTxt)
}
}
func checkMetadataSearchOrder(order string) error {
if order == "" {
return errors.New("invalid value in section [Metadata] with key `search-order`. Value cannot be empty")
}
elements := strings.Split(order, ",")
if len(elements) > 2 {
return errors.New("invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements")
}
for _, id := range elements {
id = strings.TrimSpace(id)
switch id {
case configDriveID:
case metadataID:
default:
return fmt.Errorf("invalid element %q found in section [Metadata] with key `search-order`."+
"Supported elements include %q and %q", id, configDriveID, metadataID)
}
}
return nil
}

View File

@@ -0,0 +1,98 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"fmt"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack"
)
// NewNetworkV2 creates a ServiceClient that may be used with the neutron v2 API
func (os *OpenStack) NewNetworkV2() (*gophercloud.ServiceClient, error) {
network, err := openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
return nil, fmt.Errorf("failed to find network v2 endpoint for region %s: %v", os.region, err)
}
return network, nil
}
// NewComputeV2 creates a ServiceClient that may be used with the nova v2 API
func (os *OpenStack) NewComputeV2() (*gophercloud.ServiceClient, error) {
compute, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
return nil, fmt.Errorf("failed to find compute v2 endpoint for region %s: %v", os.region, err)
}
return compute, nil
}
// NewBlockStorageV1 creates a ServiceClient that may be used with the Cinder v1 API
func (os *OpenStack) NewBlockStorageV1() (*gophercloud.ServiceClient, error) {
storage, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
return nil, fmt.Errorf("unable to initialize cinder v1 client for region %s: %v", os.region, err)
}
return storage, nil
}
// NewBlockStorageV2 creates a ServiceClient that may be used with the Cinder v2 API
func (os *OpenStack) NewBlockStorageV2() (*gophercloud.ServiceClient, error) {
storage, err := openstack.NewBlockStorageV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
return nil, fmt.Errorf("unable to initialize cinder v2 client for region %s: %v", os.region, err)
}
return storage, nil
}
// NewBlockStorageV3 creates a ServiceClient that may be used with the Cinder v3 API
func (os *OpenStack) NewBlockStorageV3() (*gophercloud.ServiceClient, error) {
storage, err := openstack.NewBlockStorageV3(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
return nil, fmt.Errorf("unable to initialize cinder v3 client for region %s: %v", os.region, err)
}
return storage, nil
}
// NewLoadBalancerV2 creates a ServiceClient that may be used with the Neutron LBaaS v2 API
func (os *OpenStack) NewLoadBalancerV2() (*gophercloud.ServiceClient, error) {
var lb *gophercloud.ServiceClient
var err error
if os.lbOpts.UseOctavia {
lb, err = openstack.NewLoadBalancerV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
} else {
lb, err = openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
}
if err != nil {
return nil, fmt.Errorf("failed to find load-balancer v2 endpoint for region %s: %v", os.region, err)
}
return lb, nil
}

View File

@@ -0,0 +1,220 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"fmt"
"regexp"
"github.com/golang/glog"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
)
// Instances encapsulates an implementation of Instances for OpenStack.
type Instances struct {
compute *gophercloud.ServiceClient
opts MetadataOpts
}
// Instances returns an implementation of Instances for OpenStack.
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
glog.V(4).Info("openstack.Instances() called")
compute, err := os.NewComputeV2()
if err != nil {
glog.Errorf("unable to access compute v2 API : %v", err)
return nil, false
}
glog.V(4).Info("Claiming to support Instances")
return &Instances{
compute: compute,
opts: os.metadataOpts,
}, true
}
// CurrentNodeName implements Instances.CurrentNodeName
// Note this is *not* necessarily the same as hostname.
func (i *Instances) CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) {
md, err := getMetadata(i.opts.SearchOrder)
if err != nil {
return "", err
}
return types.NodeName(md.Name), nil
}
// AddSSHKeyToAllInstances is not implemented for OpenStack
func (i *Instances) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error {
return cloudprovider.NotImplemented
}
// NodeAddresses implements Instances.NodeAddresses
func (i *Instances) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) {
glog.V(4).Infof("NodeAddresses(%v) called", name)
addrs, err := getAddressesByName(i.compute, name)
if err != nil {
return nil, err
}
glog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs)
return addrs, nil
}
// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) {
instanceID, err := instanceIDFromProviderID(providerID)
if err != nil {
return []v1.NodeAddress{}, err
}
server, err := servers.Get(i.compute, instanceID).Extract()
if err != nil {
return []v1.NodeAddress{}, err
}
addresses, err := nodeAddresses(server)
if err != nil {
return []v1.NodeAddress{}, err
}
return addresses, nil
}
// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
func (i *Instances) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
instanceID, err := instanceIDFromProviderID(providerID)
if err != nil {
return false, err
}
server, err := servers.Get(i.compute, instanceID).Extract()
if err != nil {
if isNotFound(err) {
return false, nil
}
return false, err
}
if server.Status != "ACTIVE" {
glog.Warningf("the instance %s is not active", instanceID)
return false, nil
}
return true, nil
}
// InstanceShutdownByProviderID returns true if the instances is in safe state to detach volumes
func (i *Instances) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
return false, cloudprovider.NotImplemented
}
// InstanceID returns the kubelet's cloud provider ID.
func (os *OpenStack) InstanceID() (string, error) {
if len(os.localInstanceID) == 0 {
id, err := readInstanceID(os.metadataOpts.SearchOrder)
if err != nil {
return "", err
}
os.localInstanceID = id
}
return os.localInstanceID, nil
}
// InstanceID returns the cloud provider ID of the specified instance.
func (i *Instances) InstanceID(ctx context.Context, name types.NodeName) (string, error) {
srv, err := getServerByName(i.compute, name)
if err != nil {
if err == ErrNotFound {
return "", cloudprovider.InstanceNotFound
}
return "", err
}
// In the future it is possible to also return an endpoint as:
// <endpoint>/<instanceid>
return "/" + srv.ID, nil
}
// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) {
instanceID, err := instanceIDFromProviderID(providerID)
if err != nil {
return "", err
}
server, err := servers.Get(i.compute, instanceID).Extract()
if err != nil {
return "", err
}
return srvInstanceType(server)
}
// InstanceType returns the type of the specified instance.
func (i *Instances) InstanceType(ctx context.Context, name types.NodeName) (string, error) {
srv, err := getServerByName(i.compute, name)
if err != nil {
return "", err
}
return srvInstanceType(srv)
}
func srvInstanceType(srv *servers.Server) (string, error) {
keys := []string{"name", "id", "original_name"}
for _, key := range keys {
val, found := srv.Flavor[key]
if found {
flavor, ok := val.(string)
if ok {
return flavor, nil
}
}
}
return "", fmt.Errorf("flavor name/id not found")
}
// instanceIDFromProviderID splits a provider's id and return instanceID.
// A providerID is build out of '${ProviderName}:///${instance-id}'which contains ':///'.
// See cloudprovider.GetInstanceProviderID and Instances.InstanceID.
func instanceIDFromProviderID(providerID string) (instanceID string, err error) {
// If Instances.InstanceID or cloudprovider.GetInstanceProviderID is changed, the regexp should be changed too.
var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `:///([^/]+)$`)
matches := providerIDRegexp.FindStringSubmatch(providerID)
if len(matches) != 2 {
return "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"openstack:///InstanceID\"", providerID)
}
return matches[1], nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,50 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import "github.com/prometheus/client_golang/prometheus"
const (
openstackSubsystem = "openstack"
openstackOperationKey = "cloudprovider_openstack_api_request_duration_seconds"
openstackOperationErrorKey = "cloudprovider_openstack_api_request_errors"
)
var (
openstackOperationsLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: openstackSubsystem,
Name: openstackOperationKey,
Help: "Latency of openstack api call",
},
[]string{"request"},
)
openstackAPIRequestErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: openstackSubsystem,
Name: openstackOperationErrorKey,
Help: "Cumulative number of openstack Api call errors",
},
[]string{"request"},
)
)
func registerMetrics() {
prometheus.MustRegister(openstackOperationsLatency)
prometheus.MustRegister(openstackAPIRequestErrors)
}

View File

@@ -0,0 +1,342 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"errors"
"net"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
)
var errNoRouterID = errors.New("router-id not set in cloud provider config")
// Routes implements the cloudprovider.Routes for OpenStack clouds
type Routes struct {
compute *gophercloud.ServiceClient
network *gophercloud.ServiceClient
opts RouterOpts
}
// NewRoutes creates a new instance of Routes
func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, opts RouterOpts) (cloudprovider.Routes, error) {
if opts.RouterID == "" {
return nil, errNoRouterID
}
return &Routes{
compute: compute,
network: network,
opts: opts,
}, nil
}
// ListRoutes lists all managed routes that belong to the specified clusterName
func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
glog.V(4).Infof("ListRoutes(%v)", clusterName)
nodeNamesByAddr := make(map[string]types.NodeName)
err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) {
addrs, err := nodeAddresses(srv)
if err != nil {
return false, err
}
name := mapServerToNodeName(srv)
for _, addr := range addrs {
nodeNamesByAddr[addr.Address] = name
}
return true, nil
})
if err != nil {
return nil, err
}
router, err := routers.Get(r.network, r.opts.RouterID).Extract()
if err != nil {
return nil, err
}
var routes []*cloudprovider.Route
for _, item := range router.Routes {
nodeName, foundNode := nodeNamesByAddr[item.NextHop]
if !foundNode {
nodeName = types.NodeName(item.NextHop)
}
route := cloudprovider.Route{
Name: item.DestinationCIDR,
TargetNode: nodeName, //contains the nexthop address if node was not found
Blackhole: !foundNode,
DestinationCIDR: item.DestinationCIDR,
}
routes = append(routes, &route)
}
return routes, nil
}
func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) {
origRoutes := router.Routes // shallow copy
_, err := routers.Update(network, router.ID, routers.UpdateOpts{
Routes: newRoutes,
}).Extract()
if err != nil {
return nil, err
}
unwinder := func() {
glog.V(4).Info("Reverting routes change to router ", router.ID)
_, err := routers.Update(network, router.ID, routers.UpdateOpts{
Routes: origRoutes,
}).Extract()
if err != nil {
glog.Warning("Unable to reset routes during error unwind: ", err)
}
}
return unwinder, nil
}
func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) {
origPairs := port.AllowedAddressPairs // shallow copy
_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
AllowedAddressPairs: &newPairs,
}).Extract()
if err != nil {
return nil, err
}
unwinder := func() {
glog.V(4).Info("Reverting allowed-address-pairs change to port ", port.ID)
_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
AllowedAddressPairs: &origPairs,
}).Extract()
if err != nil {
glog.Warning("Unable to reset allowed-address-pairs during error unwind: ", err)
}
}
return unwinder, nil
}
// CreateRoute creates the described managed route
func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error {
glog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route)
onFailure := newCaller()
ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
isCIDRv6 := ip.To4() == nil
addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6)
if err != nil {
return err
}
glog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode)
router, err := routers.Get(r.network, r.opts.RouterID).Extract()
if err != nil {
return err
}
routes := router.Routes
for _, item := range routes {
if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr {
glog.V(4).Infof("Skipping existing route: %v", route)
return nil
}
}
routes = append(routes, routers.Route{
DestinationCIDR: route.DestinationCIDR,
NextHop: addr,
})
unwind, err := updateRoutes(r.network, router, routes)
if err != nil {
return err
}
defer onFailure.call(unwind)
// get the port of addr on target node.
portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
if err != nil {
return err
}
port, err := getPortByID(r.network, portID)
if err != nil {
return err
}
found := false
for _, item := range port.AllowedAddressPairs {
if item.IPAddress == route.DestinationCIDR {
glog.V(4).Info("Found existing allowed-address-pair: ", item)
found = true
break
}
}
if !found {
newPairs := append(port.AllowedAddressPairs, neutronports.AddressPair{
IPAddress: route.DestinationCIDR,
})
unwind, err := updateAllowedAddressPairs(r.network, port, newPairs)
if err != nil {
return err
}
defer onFailure.call(unwind)
}
glog.V(4).Infof("Route created: %v", route)
onFailure.disarm()
return nil
}
// DeleteRoute deletes the specified managed route
func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error {
glog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route)
onFailure := newCaller()
ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
isCIDRv6 := ip.To4() == nil
var addr string
// Blackhole routes are orphaned and have no counterpart in OpenStack
if !route.Blackhole {
var err error
addr, err = getAddressByName(r.compute, route.TargetNode, isCIDRv6)
if err != nil {
return err
}
}
router, err := routers.Get(r.network, r.opts.RouterID).Extract()
if err != nil {
return err
}
routes := router.Routes
index := -1
for i, item := range routes {
if item.DestinationCIDR == route.DestinationCIDR && (item.NextHop == addr || route.Blackhole && item.NextHop == string(route.TargetNode)) {
index = i
break
}
}
if index == -1 {
glog.V(4).Infof("Skipping non-existent route: %v", route)
return nil
}
// Delete element `index`
routes[index] = routes[len(routes)-1]
routes = routes[:len(routes)-1]
unwind, err := updateRoutes(r.network, router, routes)
// If this was a blackhole route we are done, there are no ports to update
if err != nil || route.Blackhole {
return err
}
defer onFailure.call(unwind)
// get the port of addr on target node.
portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
if err != nil {
return err
}
port, err := getPortByID(r.network, portID)
if err != nil {
return err
}
addrPairs := port.AllowedAddressPairs
index = -1
for i, item := range addrPairs {
if item.IPAddress == route.DestinationCIDR {
index = i
break
}
}
if index != -1 {
// Delete element `index`
addrPairs[index] = addrPairs[len(addrPairs)-1]
addrPairs = addrPairs[:len(addrPairs)-1]
unwind, err := updateAllowedAddressPairs(r.network, port, addrPairs)
if err != nil {
return err
}
defer onFailure.call(unwind)
}
glog.V(4).Infof("Route deleted: %v", route)
onFailure.disarm()
return nil
}
func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName, ipAddress string) (string, error) {
srv, err := getServerByName(compute, targetNode)
if err != nil {
return "", err
}
interfaces, err := getAttachedInterfacesByID(compute, srv.ID)
if err != nil {
return "", err
}
for _, intf := range interfaces {
for _, fixedIP := range intf.FixedIPs {
if fixedIP.IPAddress == ipAddress {
return intf.PortID, nil
}
}
}
return "", ErrNotFound
}
func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) {
targetPort, err := neutronports.Get(client, portID).Extract()
if err != nil {
return nil, err
}
if targetPort == nil {
return nil, ErrNotFound
}
return targetPort, nil
}

View File

@@ -0,0 +1,122 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"net"
"testing"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
)
func TestRoutes(t *testing.T) {
const clusterName = "ignored"
cfg, ok := configFromEnv()
if !ok {
t.Skipf("No config found in environment")
}
os, err := newOpenStack(cfg)
if err != nil {
t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
}
vms := getServers(os)
_, err = os.InstanceID()
if err != nil || len(vms) == 0 {
t.Skipf("Please run this test in an OpenStack vm or create at least one VM in OpenStack before you run this test.")
}
// We know we have at least one vm.
servername := vms[0].Name
// Pick the first router and server to try a test with
os.routeOpts.RouterID = getRouters(os)[0].ID
r, ok := os.Routes()
if !ok {
t.Skip("Routes() returned false - perhaps your stack does not support Neutron extraroute extension?")
}
newroute := cloudprovider.Route{
DestinationCIDR: "10.164.2.0/24",
TargetNode: types.NodeName(servername),
}
err = r.CreateRoute(context.TODO(), clusterName, "myhint", &newroute)
if err != nil {
t.Fatalf("CreateRoute error: %v", err)
}
routelist, err := r.ListRoutes(context.TODO(), clusterName)
if err != nil {
t.Fatalf("ListRoutes() error: %v", err)
}
for _, route := range routelist {
_, cidr, err := net.ParseCIDR(route.DestinationCIDR)
if err != nil {
t.Logf("Ignoring route %s, unparsable CIDR: %v", route.Name, err)
continue
}
t.Logf("%s via %s", cidr, route.TargetNode)
}
err = r.DeleteRoute(context.TODO(), clusterName, &newroute)
if err != nil {
t.Fatalf("DeleteRoute error: %v", err)
}
}
func getServers(os *OpenStack) []servers.Server {
c, err := os.NewComputeV2()
allPages, err := servers.List(c, servers.ListOpts{}).AllPages()
if err != nil {
panic(err)
}
allServers, err := servers.ExtractServers(allPages)
if err != nil {
panic(err)
}
if len(allServers) == 0 {
panic("No servers to test with")
}
return allServers
}
func getRouters(os *OpenStack) []routers.Router {
listOpts := routers.ListOpts{}
n, err := os.NewNetworkV2()
if err != nil {
panic(err)
}
allPages, err := routers.List(n, listOpts).AllPages()
if err != nil {
panic(err)
}
allRouters, err := routers.ExtractRouters(allPages)
if err != nil {
panic(err)
}
if len(allRouters) == 0 {
panic("No routers to test with")
}
return allRouters
}

View File

@@ -0,0 +1,709 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"fmt"
"os"
"reflect"
"regexp"
"sort"
"strings"
"testing"
"time"
"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
)
const (
testClusterName = "testCluster"
volumeStatusTimeoutSeconds = 30
// volumeStatus* is configuration of exponential backoff for
// waiting for specified volume status. Starting with 1
// seconds, multiplying by 1.2 with each step and taking 13 steps at maximum
// it will time out after 32s, which roughly corresponds to 30s
volumeStatusInitDelay = 1 * time.Second
volumeStatusFactor = 1.2
volumeStatusSteps = 13
)
func WaitForVolumeStatus(t *testing.T, os *OpenStack, volumeName string, status string) {
backoff := wait.Backoff{
Duration: volumeStatusInitDelay,
Factor: volumeStatusFactor,
Steps: volumeStatusSteps,
}
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
getVol, err := os.getVolume(volumeName)
if err != nil {
return false, err
}
if getVol.Status == status {
t.Logf("Volume (%s) status changed to %s after %v seconds\n",
volumeName,
status,
volumeStatusTimeoutSeconds)
return true, nil
}
return false, nil
})
if err == wait.ErrWaitTimeout {
t.Logf("Volume (%s) status did not change to %s after %v seconds\n",
volumeName,
status,
volumeStatusTimeoutSeconds)
return
}
if err != nil {
t.Fatalf("Cannot get existing Cinder volume (%s): %v", volumeName, err)
}
}
func TestReadConfig(t *testing.T) {
_, err := readConfig(nil)
if err == nil {
t.Errorf("Should fail when no config is provided: %s", err)
}
// Since we are setting env vars, we need to reset old
// values for other tests to succeed.
env := clearEnviron(t)
defer resetEnviron(t, env)
os.Setenv("OS_PASSWORD", "mypass")
defer os.Unsetenv("OS_PASSWORD")
os.Setenv("OS_TENANT_NAME", "admin")
defer os.Unsetenv("OS_TENANT_NAME")
cfg, err := readConfig(strings.NewReader(`
[Global]
auth-url = http://auth.url
user-id = user
tenant-name = demo
[LoadBalancer]
create-monitor = yes
monitor-delay = 1m
monitor-timeout = 30s
monitor-max-retries = 3
[BlockStorage]
bs-version = auto
trust-device-path = yes
ignore-volume-az = yes
[Metadata]
search-order = configDrive, metadataService
`))
if err != nil {
t.Fatalf("Should succeed when a valid config is provided: %s", err)
}
if cfg.Global.AuthURL != "http://auth.url" {
t.Errorf("incorrect authurl: %s", cfg.Global.AuthURL)
}
if cfg.Global.UserID != "user" {
t.Errorf("incorrect userid: %s", cfg.Global.UserID)
}
if cfg.Global.Password != "mypass" {
t.Errorf("incorrect password: %s", cfg.Global.Password)
}
// config file wins over environment variable
if cfg.Global.TenantName != "demo" {
t.Errorf("incorrect tenant name: %s", cfg.Global.TenantName)
}
if !cfg.LoadBalancer.CreateMonitor {
t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor)
}
if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute {
t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay)
}
if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second {
t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout)
}
if cfg.LoadBalancer.MonitorMaxRetries != 3 {
t.Errorf("incorrect lb.monitormaxretries: %d", cfg.LoadBalancer.MonitorMaxRetries)
}
if cfg.BlockStorage.TrustDevicePath != true {
t.Errorf("incorrect bs.trustdevicepath: %v", cfg.BlockStorage.TrustDevicePath)
}
if cfg.BlockStorage.BSVersion != "auto" {
t.Errorf("incorrect bs.bs-version: %v", cfg.BlockStorage.BSVersion)
}
if cfg.BlockStorage.IgnoreVolumeAZ != true {
t.Errorf("incorrect bs.IgnoreVolumeAZ: %v", cfg.BlockStorage.IgnoreVolumeAZ)
}
if cfg.Metadata.SearchOrder != "configDrive, metadataService" {
t.Errorf("incorrect md.search-order: %v", cfg.Metadata.SearchOrder)
}
}
func TestToAuthOptions(t *testing.T) {
cfg := Config{}
cfg.Global.Username = "user"
cfg.Global.Password = "pass"
cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a"
cfg.Global.DomainName = "local"
cfg.Global.AuthURL = "http://auth.url"
cfg.Global.UserID = "user"
ao := cfg.toAuthOptions()
if !ao.AllowReauth {
t.Errorf("Will need to be able to reauthenticate")
}
if ao.Username != cfg.Global.Username {
t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username)
}
if ao.Password != cfg.Global.Password {
t.Errorf("Password %s != %s", ao.Password, cfg.Global.Password)
}
if ao.DomainID != cfg.Global.DomainID {
t.Errorf("DomainID %s != %s", ao.DomainID, cfg.Global.DomainID)
}
if ao.IdentityEndpoint != cfg.Global.AuthURL {
t.Errorf("IdentityEndpoint %s != %s", ao.IdentityEndpoint, cfg.Global.AuthURL)
}
if ao.UserID != cfg.Global.UserID {
t.Errorf("UserID %s != %s", ao.UserID, cfg.Global.UserID)
}
if ao.DomainName != cfg.Global.DomainName {
t.Errorf("DomainName %s != %s", ao.DomainName, cfg.Global.DomainName)
}
if ao.TenantID != cfg.Global.TenantID {
t.Errorf("TenantID %s != %s", ao.TenantID, cfg.Global.TenantID)
}
}
func TestCheckOpenStackOpts(t *testing.T) {
delay := MyDuration{60 * time.Second}
timeout := MyDuration{30 * time.Second}
tests := []struct {
name string
openstackOpts *OpenStack
expectedError error
}{
{
name: "test1",
openstackOpts: &OpenStack{
provider: nil,
lbOpts: LoadBalancerOpts{
LBVersion: "v2",
SubnetID: "6261548e-ffde-4bc7-bd22-59c83578c5ef",
FloatingNetworkID: "38b8b5f9-64dc-4424-bf86-679595714786",
LBMethod: "ROUND_ROBIN",
LBProvider: "haproxy",
CreateMonitor: true,
MonitorDelay: delay,
MonitorTimeout: timeout,
MonitorMaxRetries: uint(3),
ManageSecurityGroups: true,
},
metadataOpts: MetadataOpts{
SearchOrder: configDriveID,
},
},
expectedError: nil,
},
{
name: "test2",
openstackOpts: &OpenStack{
provider: nil,
lbOpts: LoadBalancerOpts{
LBVersion: "v2",
FloatingNetworkID: "38b8b5f9-64dc-4424-bf86-679595714786",
LBMethod: "ROUND_ROBIN",
CreateMonitor: true,
MonitorDelay: delay,
MonitorTimeout: timeout,
MonitorMaxRetries: uint(3),
ManageSecurityGroups: true,
},
metadataOpts: MetadataOpts{
SearchOrder: configDriveID,
},
},
expectedError: nil,
},
{
name: "test3",
openstackOpts: &OpenStack{
provider: nil,
lbOpts: LoadBalancerOpts{
LBVersion: "v2",
SubnetID: "6261548e-ffde-4bc7-bd22-59c83578c5ef",
FloatingNetworkID: "38b8b5f9-64dc-4424-bf86-679595714786",
LBMethod: "ROUND_ROBIN",
CreateMonitor: true,
MonitorTimeout: timeout,
MonitorMaxRetries: uint(3),
ManageSecurityGroups: true,
},
metadataOpts: MetadataOpts{
SearchOrder: configDriveID,
},
},
expectedError: fmt.Errorf("monitor-delay not set in cloud provider config"),
},
{
name: "test4",
openstackOpts: &OpenStack{
provider: nil,
metadataOpts: MetadataOpts{
SearchOrder: "",
},
},
expectedError: fmt.Errorf("invalid value in section [Metadata] with key `search-order`. Value cannot be empty"),
},
{
name: "test5",
openstackOpts: &OpenStack{
provider: nil,
metadataOpts: MetadataOpts{
SearchOrder: "value1,value2,value3",
},
},
expectedError: fmt.Errorf("invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements"),
},
{
name: "test6",
openstackOpts: &OpenStack{
provider: nil,
metadataOpts: MetadataOpts{
SearchOrder: "value1",
},
},
expectedError: fmt.Errorf("invalid element %q found in section [Metadata] with key `search-order`."+
"Supported elements include %q and %q", "value1", configDriveID, metadataID),
},
{
name: "test7",
openstackOpts: &OpenStack{
provider: nil,
lbOpts: LoadBalancerOpts{
LBVersion: "v2",
SubnetID: "6261548e-ffde-4bc7-bd22-59c83578c5ef",
FloatingNetworkID: "38b8b5f9-64dc-4424-bf86-679595714786",
LBMethod: "ROUND_ROBIN",
CreateMonitor: true,
MonitorDelay: delay,
MonitorTimeout: timeout,
ManageSecurityGroups: true,
},
metadataOpts: MetadataOpts{
SearchOrder: configDriveID,
},
},
expectedError: fmt.Errorf("monitor-max-retries not set in cloud provider config"),
},
{
name: "test8",
openstackOpts: &OpenStack{
provider: nil,
lbOpts: LoadBalancerOpts{
LBVersion: "v2",
SubnetID: "6261548e-ffde-4bc7-bd22-59c83578c5ef",
FloatingNetworkID: "38b8b5f9-64dc-4424-bf86-679595714786",
LBMethod: "ROUND_ROBIN",
CreateMonitor: true,
MonitorDelay: delay,
MonitorMaxRetries: uint(3),
ManageSecurityGroups: true,
},
metadataOpts: MetadataOpts{
SearchOrder: configDriveID,
},
},
expectedError: fmt.Errorf("monitor-timeout not set in cloud provider config"),
},
}
for _, testcase := range tests {
err := checkOpenStackOpts(testcase.openstackOpts)
if err == nil && testcase.expectedError == nil {
continue
}
if (err != nil && testcase.expectedError == nil) || (err == nil && testcase.expectedError != nil) || err.Error() != testcase.expectedError.Error() {
t.Errorf("%s failed: expected err=%q, got %q",
testcase.name, testcase.expectedError, err)
}
}
}
func TestCaller(t *testing.T) {
called := false
myFunc := func() { called = true }
c := newCaller()
c.call(myFunc)
if !called {
t.Errorf("caller failed to call function in default case")
}
c.disarm()
called = false
c.call(myFunc)
if called {
t.Error("caller still called function when disarmed")
}
// Confirm the "usual" deferred caller pattern works as expected
called = false
successCase := func() {
c := newCaller()
defer c.call(func() { called = true })
c.disarm()
}
if successCase(); called {
t.Error("Deferred success case still invoked unwind")
}
called = false
failureCase := func() {
c := newCaller()
defer c.call(func() { called = true })
}
if failureCase(); !called {
t.Error("Deferred failure case failed to invoke unwind")
}
}
// An arbitrary sort.Interface, just for easier comparison
type AddressSlice []v1.NodeAddress
func (a AddressSlice) Len() int { return len(a) }
func (a AddressSlice) Less(i, j int) bool { return a[i].Address < a[j].Address }
func (a AddressSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func TestNodeAddresses(t *testing.T) {
srv := servers.Server{
Status: "ACTIVE",
HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362",
AccessIPv4: "50.56.176.99",
AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8",
Addresses: map[string]interface{}{
"private": []interface{}{
map[string]interface{}{
"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b",
"version": float64(4),
"addr": "10.0.0.32",
"OS-EXT-IPS:type": "fixed",
},
map[string]interface{}{
"version": float64(4),
"addr": "50.56.176.36",
"OS-EXT-IPS:type": "floating",
},
map[string]interface{}{
"version": float64(4),
"addr": "10.0.0.31",
// No OS-EXT-IPS:type
},
},
"public": []interface{}{
map[string]interface{}{
"version": float64(4),
"addr": "50.56.176.35",
},
map[string]interface{}{
"version": float64(6),
"addr": "2001:4800:780e:510:be76:4eff:fe04:84a8",
},
},
},
}
addrs, err := nodeAddresses(&srv)
if err != nil {
t.Fatalf("nodeAddresses returned error: %v", err)
}
sort.Sort(AddressSlice(addrs))
t.Logf("addresses is %v", addrs)
want := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
}
if !reflect.DeepEqual(want, addrs) {
t.Errorf("nodeAddresses returned incorrect value %v", addrs)
}
}
func TestNewOpenStack(t *testing.T) {
cfg, ok := configFromEnv()
if !ok {
t.Skip("No config found in environment")
}
_, err := newOpenStack(cfg)
if err != nil {
t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
}
}
func TestLoadBalancer(t *testing.T) {
cfg, ok := configFromEnv()
if !ok {
t.Skip("No config found in environment")
}
versions := []string{"v2", ""}
for _, v := range versions {
t.Logf("Trying LBVersion = '%s'\n", v)
cfg.LoadBalancer.LBVersion = v
os, err := newOpenStack(cfg)
if err != nil {
t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
}
lb, ok := os.LoadBalancer()
if !ok {
t.Fatalf("LoadBalancer() returned false - perhaps your stack doesn't support Neutron?")
}
_, exists, err := lb.GetLoadBalancer(context.TODO(), testClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}})
if err != nil {
t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err)
}
if exists {
t.Fatalf("GetLoadBalancer(\"noexist\") returned exists")
}
}
}
func TestZones(t *testing.T) {
SetMetadataFixture(&FakeMetadata)
defer ClearMetadata()
os := OpenStack{
provider: &gophercloud.ProviderClient{
IdentityBase: "http://auth.url/",
},
region: "myRegion",
}
z, ok := os.Zones()
if !ok {
t.Fatalf("Zones() returned false")
}
zone, err := z.GetZone(context.TODO())
if err != nil {
t.Fatalf("GetZone() returned error: %s", err)
}
if zone.Region != "myRegion" {
t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
}
if zone.FailureDomain != "nova" {
t.Fatalf("GetZone() returned wrong failure domain (%s)", zone.FailureDomain)
}
}
var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/")
func TestVolumes(t *testing.T) {
cfg, ok := configFromEnv()
if !ok {
t.Skip("No config found in environment")
}
os, err := newOpenStack(cfg)
if err != nil {
t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
}
tags := map[string]string{
"test": "value",
}
vol, _, _, err := os.CreateVolume("kubernetes-test-volume-"+rand.String(10), 1, "", "", &tags)
if err != nil {
t.Fatalf("Cannot create a new Cinder volume: %v", err)
}
t.Logf("Volume (%s) created\n", vol)
WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
id, err := os.InstanceID()
if err != nil {
t.Logf("Cannot find instance id: %v - perhaps you are running this test outside a VM launched by OpenStack", err)
} else {
diskID, err := os.AttachDisk(id, vol)
if err != nil {
t.Fatalf("Cannot AttachDisk Cinder volume %s: %v", vol, err)
}
t.Logf("Volume (%s) attached, disk ID: %s\n", vol, diskID)
WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
devicePath := os.GetDevicePath(diskID)
if diskPathRegexp.FindString(devicePath) == "" {
t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
}
t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)
err = os.DetachDisk(id, vol)
if err != nil {
t.Fatalf("Cannot DetachDisk Cinder volume %s: %v", vol, err)
}
t.Logf("Volume (%s) detached\n", vol)
WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
}
expectedVolSize := resource.MustParse("2Gi")
newVolSize, err := os.ExpandVolume(vol, resource.MustParse("1Gi"), expectedVolSize)
if err != nil {
t.Fatalf("Cannot expand a Cinder volume: %v", err)
}
if newVolSize != expectedVolSize {
t.Logf("Expected: %v but got: %v ", expectedVolSize, newVolSize)
}
t.Logf("Volume expanded to (%v) \n", newVolSize)
WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
err = os.DeleteVolume(vol)
if err != nil {
t.Fatalf("Cannot delete Cinder volume %s: %v", vol, err)
}
t.Logf("Volume (%s) deleted\n", vol)
}
func TestInstanceIDFromProviderID(t *testing.T) {
testCases := []struct {
providerID string
instanceID string
fail bool
}{
{
providerID: ProviderName + "://" + "/" + "7b9cf879-7146-417c-abfd-cb4272f0c935",
instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
fail: false,
},
{
providerID: "openstack://7b9cf879-7146-417c-abfd-cb4272f0c935",
instanceID: "",
fail: true,
},
{
providerID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
instanceID: "",
fail: true,
},
{
providerID: "other-provider:///7b9cf879-7146-417c-abfd-cb4272f0c935",
instanceID: "",
fail: true,
},
}
for _, test := range testCases {
instanceID, err := instanceIDFromProviderID(test.providerID)
if (err != nil) != test.fail {
t.Errorf("%s yielded `err != nil` as %t. expected %t", test.providerID, (err != nil), test.fail)
}
if test.fail {
continue
}
if instanceID != test.instanceID {
t.Errorf("%s yielded %s. expected %s", test.providerID, instanceID, test.instanceID)
}
}
}
func TestToAuth3Options(t *testing.T) {
cfg := Config{}
cfg.Global.Username = "user"
cfg.Global.Password = "pass"
cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a"
cfg.Global.DomainName = "local"
cfg.Global.AuthURL = "http://auth.url"
cfg.Global.UserID = "user"
ao := cfg.toAuth3Options()
if !ao.AllowReauth {
t.Errorf("Will need to be able to reauthenticate")
}
if ao.Username != cfg.Global.Username {
t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username)
}
if ao.Password != cfg.Global.Password {
t.Errorf("Password %s != %s", ao.Password, cfg.Global.Password)
}
if ao.DomainID != cfg.Global.DomainID {
t.Errorf("DomainID %s != %s", ao.DomainID, cfg.Global.DomainID)
}
if ao.IdentityEndpoint != cfg.Global.AuthURL {
t.Errorf("IdentityEndpoint %s != %s", ao.IdentityEndpoint, cfg.Global.AuthURL)
}
if ao.UserID != cfg.Global.UserID {
t.Errorf("UserID %s != %s", ao.UserID, cfg.Global.UserID)
}
if ao.DomainName != cfg.Global.DomainName {
t.Errorf("DomainName %s != %s", ao.DomainName, cfg.Global.DomainName)
}
}
func clearEnviron(t *testing.T) []string {
env := os.Environ()
for _, pair := range env {
if strings.HasPrefix(pair, "OS_") {
i := strings.Index(pair, "=") + 1
os.Unsetenv(pair[:i-1])
}
}
return env
}
func resetEnviron(t *testing.T, items []string) {
for _, pair := range items {
if strings.HasPrefix(pair, "OS_") {
i := strings.Index(pair, "=") + 1
if err := os.Setenv(pair[:i-1], pair[i:]); err != nil {
t.Errorf("Setenv(%q, %q) failed during reset: %v", pair[:i-1], pair[i:], err)
}
}
}
}

View File

@@ -0,0 +1,724 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package openstack
import (
"context"
"errors"
"fmt"
"io/ioutil"
"path"
"path/filepath"
"strings"
"time"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
k8s_volume "k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"github.com/gophercloud/gophercloud"
volumeexpand "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
volumes_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes"
volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
volumes_v3 "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
"github.com/prometheus/client_golang/prometheus"
"github.com/golang/glog"
)
type volumeService interface {
createVolume(opts volumeCreateOpts) (string, string, error)
getVolume(volumeID string) (Volume, error)
deleteVolume(volumeName string) error
expandVolume(volumeID string, newSize int) error
}
// VolumesV1 is a Volumes implementation for cinder v1
type VolumesV1 struct {
blockstorage *gophercloud.ServiceClient
opts BlockStorageOpts
}
// VolumesV2 is a Volumes implementation for cinder v2
type VolumesV2 struct {
blockstorage *gophercloud.ServiceClient
opts BlockStorageOpts
}
// VolumesV3 is a Volumes implementation for cinder v3
type VolumesV3 struct {
blockstorage *gophercloud.ServiceClient
opts BlockStorageOpts
}
// Volume stores information about a single volume
type Volume struct {
// ID of the instance, to which this volume is attached. "" if not attached
AttachedServerID string
// Device file path
AttachedDevice string
// availabilityZone is which availability zone the volume is in
AvailabilityZone string
// Unique identifier for the volume.
ID string
// Human-readable display name for the volume.
Name string
// Current status of the volume.
Status string
// Volume size in GB
Size int
}
type volumeCreateOpts struct {
Size int
Availability string
Name string
VolumeType string
Metadata map[string]string
}
// implements PVLabeler.
var _ cloudprovider.PVLabeler = (*OpenStack)(nil)
const (
volumeAvailableStatus = "available"
volumeInUseStatus = "in-use"
volumeDeletedStatus = "deleted"
volumeErrorStatus = "error"
// On some environments, we need to query the metadata service in order
// to locate disks. We'll use the Newton version, which includes device
// metadata.
newtonMetadataVersion = "2016-06-30"
)
func (volumes *VolumesV1) createVolume(opts volumeCreateOpts) (string, string, error) {
startTime := time.Now()
createOpts := volumes_v1.CreateOpts{
Name: opts.Name,
Size: opts.Size,
VolumeType: opts.VolumeType,
AvailabilityZone: opts.Availability,
Metadata: opts.Metadata,
}
vol, err := volumes_v1.Create(volumes.blockstorage, createOpts).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("create_v1_volume", timeTaken, err)
if err != nil {
return "", "", err
}
return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV2) createVolume(opts volumeCreateOpts) (string, string, error) {
startTime := time.Now()
createOpts := volumes_v2.CreateOpts{
Name: opts.Name,
Size: opts.Size,
VolumeType: opts.VolumeType,
AvailabilityZone: opts.Availability,
Metadata: opts.Metadata,
}
vol, err := volumes_v2.Create(volumes.blockstorage, createOpts).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("create_v2_volume", timeTaken, err)
if err != nil {
return "", "", err
}
return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV3) createVolume(opts volumeCreateOpts) (string, string, error) {
startTime := time.Now()
createOpts := volumes_v3.CreateOpts{
Name: opts.Name,
Size: opts.Size,
VolumeType: opts.VolumeType,
AvailabilityZone: opts.Availability,
Metadata: opts.Metadata,
}
vol, err := volumes_v3.Create(volumes.blockstorage, createOpts).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("create_v3_volume", timeTaken, err)
if err != nil {
return "", "", err
}
return vol.ID, vol.AvailabilityZone, nil
}
func (volumes *VolumesV1) getVolume(volumeID string) (Volume, error) {
startTime := time.Now()
volumeV1, err := volumes_v1.Get(volumes.blockstorage, volumeID).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("get_v1_volume", timeTaken, err)
if err != nil {
return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
}
volume := Volume{
AvailabilityZone: volumeV1.AvailabilityZone,
ID: volumeV1.ID,
Name: volumeV1.Name,
Status: volumeV1.Status,
Size: volumeV1.Size,
}
if len(volumeV1.Attachments) > 0 && volumeV1.Attachments[0]["server_id"] != nil {
volume.AttachedServerID = volumeV1.Attachments[0]["server_id"].(string)
volume.AttachedDevice = volumeV1.Attachments[0]["device"].(string)
}
return volume, nil
}
func (volumes *VolumesV2) getVolume(volumeID string) (Volume, error) {
startTime := time.Now()
volumeV2, err := volumes_v2.Get(volumes.blockstorage, volumeID).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("get_v2_volume", timeTaken, err)
if err != nil {
return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
}
volume := Volume{
AvailabilityZone: volumeV2.AvailabilityZone,
ID: volumeV2.ID,
Name: volumeV2.Name,
Status: volumeV2.Status,
Size: volumeV2.Size,
}
if len(volumeV2.Attachments) > 0 {
volume.AttachedServerID = volumeV2.Attachments[0].ServerID
volume.AttachedDevice = volumeV2.Attachments[0].Device
}
return volume, nil
}
func (volumes *VolumesV3) getVolume(volumeID string) (Volume, error) {
startTime := time.Now()
volumeV3, err := volumes_v3.Get(volumes.blockstorage, volumeID).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("get_v3_volume", timeTaken, err)
if err != nil {
return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
}
volume := Volume{
AvailabilityZone: volumeV3.AvailabilityZone,
ID: volumeV3.ID,
Name: volumeV3.Name,
Status: volumeV3.Status,
}
if len(volumeV3.Attachments) > 0 {
volume.AttachedServerID = volumeV3.Attachments[0].ServerID
volume.AttachedDevice = volumeV3.Attachments[0].Device
}
return volume, nil
}
func (volumes *VolumesV1) deleteVolume(volumeID string) error {
startTime := time.Now()
err := volumes_v1.Delete(volumes.blockstorage, volumeID).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("delete_v1_volume", timeTaken, err)
return err
}
func (volumes *VolumesV2) deleteVolume(volumeID string) error {
startTime := time.Now()
err := volumes_v2.Delete(volumes.blockstorage, volumeID).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("delete_v2_volume", timeTaken, err)
return err
}
func (volumes *VolumesV3) deleteVolume(volumeID string) error {
startTime := time.Now()
err := volumes_v3.Delete(volumes.blockstorage, volumeID).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("delete_v3_volume", timeTaken, err)
return err
}
func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
startTime := time.Now()
createOpts := volumeexpand.ExtendSizeOpts{
NewSize: newSize,
}
err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("expand_volume", timeTaken, err)
return err
}
func (volumes *VolumesV2) expandVolume(volumeID string, newSize int) error {
startTime := time.Now()
createOpts := volumeexpand.ExtendSizeOpts{
NewSize: newSize,
}
err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("expand_volume", timeTaken, err)
return err
}
func (volumes *VolumesV3) expandVolume(volumeID string, newSize int) error {
startTime := time.Now()
createOpts := volumeexpand.ExtendSizeOpts{
NewSize: newSize,
}
err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, createOpts).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("expand_volume", timeTaken, err)
return err
}
// OperationPending checks if there is an operation pending on a volume
func (os *OpenStack) OperationPending(diskName string) (bool, string, error) {
volume, err := os.getVolume(diskName)
if err != nil {
return false, "", err
}
volumeStatus := volume.Status
if volumeStatus == volumeErrorStatus {
err = fmt.Errorf("status of volume %s is %s", diskName, volumeStatus)
return false, volumeStatus, err
}
if volumeStatus == volumeAvailableStatus || volumeStatus == volumeInUseStatus || volumeStatus == volumeDeletedStatus {
return false, volume.Status, nil
}
return true, volumeStatus, nil
}
// AttachDisk attaches given cinder volume to the compute running kubelet
func (os *OpenStack) AttachDisk(instanceID, volumeID string) (string, error) {
volume, err := os.getVolume(volumeID)
if err != nil {
return "", err
}
cClient, err := os.NewComputeV2()
if err != nil {
return "", err
}
if volume.AttachedServerID != "" {
if instanceID == volume.AttachedServerID {
glog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID)
return volume.ID, nil
}
nodeName, err := os.GetNodeNameByID(volume.AttachedServerID)
attachErr := fmt.Sprintf("disk %s path %s is attached to a different instance (%s)", volumeID, volume.AttachedDevice, volume.AttachedServerID)
if err != nil {
glog.Error(attachErr)
return "", errors.New(attachErr)
}
// using volume.AttachedDevice may cause problems because cinder does not report device path correctly see issue #33128
devicePath := volume.AttachedDevice
danglingErr := volumeutil.NewDanglingError(attachErr, nodeName, devicePath)
glog.V(2).Infof("Found dangling volume %s attached to node %s", volumeID, nodeName)
return "", danglingErr
}
startTime := time.Now()
// add read only flag here if possible spothanis
_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
VolumeID: volume.ID,
}).Extract()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("attach_disk", timeTaken, err)
if err != nil {
return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
}
glog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID)
return volume.ID, nil
}
// DetachDisk detaches given cinder volume from the compute running kubelet
func (os *OpenStack) DetachDisk(instanceID, volumeID string) error {
volume, err := os.getVolume(volumeID)
if err != nil {
return err
}
if volume.Status == volumeAvailableStatus {
// "available" is fine since that means the volume is detached from instance already.
glog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID)
return nil
}
if volume.Status != volumeInUseStatus {
return fmt.Errorf("can not detach volume %s, its status is %s", volume.Name, volume.Status)
}
cClient, err := os.NewComputeV2()
if err != nil {
return err
}
if volume.AttachedServerID != instanceID {
return fmt.Errorf("disk: %s has no attachments or is not attached to compute: %s", volume.Name, instanceID)
}
startTime := time.Now()
// This is a blocking call and effects kubelet's performance directly.
// We should consider kicking it out into a separate routine, if it is bad.
err = volumeattach.Delete(cClient, instanceID, volume.ID).ExtractErr()
timeTaken := time.Since(startTime).Seconds()
recordOpenstackOperationMetric("detach_disk", timeTaken, err)
if err != nil {
return fmt.Errorf("failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err)
}
glog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID)
return nil
}
// ExpandVolume expands the size of specific cinder volume (in GiB)
func (os *OpenStack) ExpandVolume(volumeID string, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
volume, err := os.getVolume(volumeID)
if err != nil {
return oldSize, err
}
if volume.Status != volumeAvailableStatus {
// cinder volume can not be expanded if its status is not available
return oldSize, fmt.Errorf("volume status is not available")
}
volSizeBytes := newSize.Value()
// Cinder works with gigabytes, convert to GiB with rounding up
volSizeGB := int(volumeutil.RoundUpSize(volSizeBytes, 1024*1024*1024))
newSizeQuant := resource.MustParse(fmt.Sprintf("%dGi", volSizeGB))
// if volume size equals to or greater than the newSize, return nil
if volume.Size >= volSizeGB {
return newSizeQuant, nil
}
volumes, err := os.volumeService("")
if err != nil {
return oldSize, err
}
err = volumes.expandVolume(volumeID, volSizeGB)
if err != nil {
return oldSize, err
}
return newSizeQuant, nil
}
// getVolume retrieves Volume by its ID.
func (os *OpenStack) getVolume(volumeID string) (Volume, error) {
volumes, err := os.volumeService("")
if err != nil {
return Volume{}, fmt.Errorf("unable to initialize cinder client for region: %s, err: %v", os.region, err)
}
return volumes.getVolume(volumeID)
}
// CreateVolume creates a volume of given size (in GiB)
func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (string, string, bool, error) {
volumes, err := os.volumeService("")
if err != nil {
return "", "", os.bsOpts.IgnoreVolumeAZ, fmt.Errorf("unable to initialize cinder client for region: %s, err: %v", os.region, err)
}
opts := volumeCreateOpts{
Name: name,
Size: size,
VolumeType: vtype,
Availability: availability,
}
if tags != nil {
opts.Metadata = *tags
}
volumeID, volumeAZ, err := volumes.createVolume(opts)
if err != nil {
return "", "", os.bsOpts.IgnoreVolumeAZ, fmt.Errorf("failed to create a %d GB volume: %v", size, err)
}
glog.Infof("Created volume %v in Availability Zone: %v Ignore volume AZ: %v", volumeID, volumeAZ, os.bsOpts.IgnoreVolumeAZ)
return volumeID, volumeAZ, os.bsOpts.IgnoreVolumeAZ, nil
}
// GetDevicePathBySerialID returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePathBySerialID(volumeID string) string {
// Build a list of candidate device paths.
// Certain Nova drivers will set the disk serial ID, including the Cinder volume id.
candidateDeviceNodes := []string{
// KVM
fmt.Sprintf("virtio-%s", volumeID[:20]),
// KVM virtio-scsi
fmt.Sprintf("scsi-0QEMU_QEMU_HARDDISK_%s", volumeID[:20]),
// ESXi
fmt.Sprintf("wwn-0x%s", strings.Replace(volumeID, "-", "", -1)),
}
files, _ := ioutil.ReadDir("/dev/disk/by-id/")
for _, f := range files {
for _, c := range candidateDeviceNodes {
if c == f.Name() {
glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
return path.Join("/dev/disk/by-id/", f.Name())
}
}
}
glog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID)
return ""
}
func (os *OpenStack) getDevicePathFromInstanceMetadata(volumeID string) string {
// Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate
// volumes, we're querying the metadata service. Note that the Hyper-V
// driver will include device metadata for untagged volumes as well.
//
// We're avoiding using cached metadata (or the configdrive),
// relying on the metadata service.
instanceMetadata, err := getMetadataFromMetadataService(
newtonMetadataVersion)
if err != nil {
glog.V(4).Infof(
"Could not retrieve instance metadata. Error: %v", err)
return ""
}
for _, device := range instanceMetadata.Devices {
if device.Type == "disk" && device.Serial == volumeID {
glog.V(4).Infof(
"Found disk metadata for volumeID %q. Bus: %q, Address: %q",
volumeID, device.Bus, device.Address)
diskPattern := fmt.Sprintf(
"/dev/disk/by-path/*-%s-%s",
device.Bus, device.Address)
diskPaths, err := filepath.Glob(diskPattern)
if err != nil {
glog.Errorf(
"could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v",
volumeID, diskPattern, err)
return ""
}
if len(diskPaths) == 1 {
return diskPaths[0]
}
glog.Errorf(
"expecting to find one disk path for volumeID %q, found %d: %v",
volumeID, len(diskPaths), diskPaths)
return ""
}
}
glog.V(4).Infof(
"Could not retrieve device metadata for volumeID: %q", volumeID)
return ""
}
// GetDevicePath returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePath(volumeID string) string {
devicePath := os.GetDevicePathBySerialID(volumeID)
if devicePath == "" {
devicePath = os.getDevicePathFromInstanceMetadata(volumeID)
}
if devicePath == "" {
glog.Warningf("Failed to find device for the volumeID: %q", volumeID)
}
return devicePath
}
// DeleteVolume deletes a volume given volume name.
func (os *OpenStack) DeleteVolume(volumeID string) error {
used, err := os.diskIsUsed(volumeID)
if err != nil {
return err
}
if used {
msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeID)
return k8s_volume.NewDeletedVolumeInUseError(msg)
}
volumes, err := os.volumeService("")
if err != nil {
return fmt.Errorf("unable to initialize cinder client for region: %s, err: %v", os.region, err)
}
err = volumes.deleteVolume(volumeID)
return err
}
// GetAttachmentDiskPath gets device path of attached volume to the compute running kubelet, as known by cinder
func (os *OpenStack) GetAttachmentDiskPath(instanceID, volumeID string) (string, error) {
// See issue #33128 - Cinder does not always tell you the right device path, as such
// we must only use this value as a last resort.
volume, err := os.getVolume(volumeID)
if err != nil {
return "", err
}
if volume.Status != volumeInUseStatus {
return "", fmt.Errorf("can not get device path of volume %s, its status is %s ", volume.Name, volume.Status)
}
if volume.AttachedServerID != "" {
if instanceID == volume.AttachedServerID {
// Attachment[0]["device"] points to the device path
// see http://developer.openstack.org/api-ref-blockstorage-v1.html
return volume.AttachedDevice, nil
}
return "", fmt.Errorf("disk %q is attached to a different compute: %q, should be detached before proceeding", volumeID, volume.AttachedServerID)
}
return "", fmt.Errorf("volume %s has no ServerId", volumeID)
}
// DiskIsAttached queries if a volume is attached to a compute instance
func (os *OpenStack) DiskIsAttached(instanceID, volumeID string) (bool, error) {
if instanceID == "" {
glog.Warningf("calling DiskIsAttached with empty instanceid: %s %s", instanceID, volumeID)
}
volume, err := os.getVolume(volumeID)
if err != nil {
return false, err
}
return instanceID == volume.AttachedServerID, nil
}
// DiskIsAttachedByName queries if a volume is attached to a compute instance by name
func (os *OpenStack) DiskIsAttachedByName(nodeName types.NodeName, volumeID string) (bool, string, error) {
cClient, err := os.NewComputeV2()
if err != nil {
return false, "", err
}
srv, err := getServerByName(cClient, nodeName)
if err != nil {
if err == ErrNotFound {
// instance not found anymore in cloudprovider, assume that cinder is detached
return false, "", nil
}
return false, "", err
}
instanceID := "/" + srv.ID
if ind := strings.LastIndex(instanceID, "/"); ind >= 0 {
instanceID = instanceID[(ind + 1):]
}
attached, err := os.DiskIsAttached(instanceID, volumeID)
return attached, instanceID, err
}
// DisksAreAttached queries if a list of volumes are attached to a compute instance
func (os *OpenStack) DisksAreAttached(instanceID string, volumeIDs []string) (map[string]bool, error) {
attached := make(map[string]bool)
for _, volumeID := range volumeIDs {
isAttached, err := os.DiskIsAttached(instanceID, volumeID)
if err != nil && err != ErrNotFound {
attached[volumeID] = true
continue
}
attached[volumeID] = isAttached
}
return attached, nil
}
// DisksAreAttachedByName queries if a list of volumes are attached to a compute instance by name
func (os *OpenStack) DisksAreAttachedByName(nodeName types.NodeName, volumeIDs []string) (map[string]bool, error) {
attached := make(map[string]bool)
cClient, err := os.NewComputeV2()
if err != nil {
return attached, err
}
srv, err := getServerByName(cClient, nodeName)
if err != nil {
if err == ErrNotFound {
// instance not found anymore, mark all volumes as detached
for _, volumeID := range volumeIDs {
attached[volumeID] = false
}
return attached, nil
}
return attached, err
}
instanceID := "/" + srv.ID
if ind := strings.LastIndex(instanceID, "/"); ind >= 0 {
instanceID = instanceID[(ind + 1):]
}
return os.DisksAreAttached(instanceID, volumeIDs)
}
// diskIsUsed returns true a disk is attached to any node.
func (os *OpenStack) diskIsUsed(volumeID string) (bool, error) {
volume, err := os.getVolume(volumeID)
if err != nil {
return false, err
}
return volume.AttachedServerID != "", nil
}
// ShouldTrustDevicePath queries if we should trust the cinder provide deviceName, See issue #33128
func (os *OpenStack) ShouldTrustDevicePath() bool {
return os.bsOpts.TrustDevicePath
}
// GetLabelsForVolume implements PVLabeler.GetLabelsForVolume
func (os *OpenStack) GetLabelsForVolume(ctx context.Context, pv *v1.PersistentVolume) (map[string]string, error) {
// Ignore any volumes that are being provisioned
if pv.Spec.Cinder.VolumeID == k8s_volume.ProvisionedVolumeName {
return nil, nil
}
// Get Volume
volume, err := os.getVolume(pv.Spec.Cinder.VolumeID)
if err != nil {
return nil, err
}
// Construct Volume Labels
labels := make(map[string]string)
labels[kubeletapis.LabelZoneFailureDomain] = volume.AvailabilityZone
labels[kubeletapis.LabelZoneRegion] = os.region
glog.V(4).Infof("The Volume %s has labels %v", pv.Spec.Cinder.VolumeID, labels)
return labels, nil
}
// recordOpenstackOperationMetric records openstack operation metrics
func recordOpenstackOperationMetric(operation string, timeTaken float64, err error) {
if err != nil {
openstackAPIRequestErrors.With(prometheus.Labels{"request": operation}).Inc()
} else {
openstackOperationsLatency.With(prometheus.Labels{"request": operation}).Observe(timeTaken)
}
}