Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 8 additions & 45 deletions nodeup/pkg/model/protokube.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"os"
"path/filepath"
"regexp"
"strings"

"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
Expand Down Expand Up @@ -146,22 +145,14 @@ func (t *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) {

// ProtokubeFlags are the flags for protokube
type ProtokubeFlags struct {
ClusterID *string `json:"clusterID,omitempty" flag:"cluster-id"`
Channels []string `json:"channels,omitempty" flag:"channels"`
Cloud *string `json:"cloud,omitempty" flag:"cloud"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
Gossip *bool `json:"gossip,omitempty" flag:"gossip"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`
Zone []string `json:"zone,omitempty" flag:"zone"`

// BootstrapMasterNodeLabels applies the critical node-role labels to our node,
// which lets us bring up the controllers that can only run on masters, which are then
// responsible for node labels. The node is specified by NodeName
BootstrapMasterNodeLabels bool `json:"bootstrapMasterNodeLabels,omitempty" flag:"bootstrap-master-node-labels"`

// NodeName is the name of the node as will be created in kubernetes. Primarily used by BootstrapMasterNodeLabels.
Channels []string `json:"channels,omitempty" flag:"channels"`
Cloud *string `json:"cloud,omitempty" flag:"cloud"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
Gossip *bool `json:"gossip,omitempty" flag:"gossip"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`

// NodeName is the name of the node as will be created in kubernetes.
NodeName string `json:"nodeName,omitempty" flag:"node-name"`

GossipProtocol *string `json:"gossip-protocol" flag:"gossip-protocol"`
Expand All @@ -183,23 +174,6 @@ func (t *ProtokubeBuilder) ProtokubeFlags() (*ProtokubeFlags, error) {
Master: b(t.IsMaster),
}

f.ClusterID = fi.PtrTo(t.NodeupConfig.ClusterName)

zone := t.NodeupConfig.DNSZone
if zone != "" {
if strings.Contains(zone, ".") {
// match by name
f.Zone = append(f.Zone, zone)
} else {
// match by id
f.Zone = append(f.Zone, "*/"+zone)
}
} else {
klog.Warningf("DNSZone not specified; protokube won't be able to update DNS")
// @TODO: Should we permit wildcard updates if zone is not specified?
// argv = append(argv, "--zone=*/*")
}

if t.UsesLegacyGossip() {
klog.Warningf("using (legacy) gossip DNS")
f.Gossip = fi.PtrTo(true)
Expand All @@ -214,19 +188,8 @@ func (t *ProtokubeBuilder) ProtokubeFlags() (*ProtokubeFlags, error) {
f.GossipSecretSecondary = t.NodeupConfig.GossipConfig.Secondary.Secret
}
}

// @TODO: This is hacky, but we want it so that we can have a different internal & external name
internalSuffix := t.APIInternalName()
internalSuffix = strings.TrimPrefix(internalSuffix, "api.")
f.DNSInternalSuffix = fi.PtrTo(internalSuffix)
}

if f.DNSInternalSuffix == nil {
f.DNSInternalSuffix = fi.PtrTo(".internal." + t.NodeupConfig.ClusterName)
}

f.BootstrapMasterNodeLabels = true

nodeName, err := t.NodeName()
if err != nil {
return nil, fmt.Errorf("error getting NodeName: %v", err)
Expand Down
2 changes: 1 addition & 1 deletion nodeup/pkg/model/tests/protokube/tasks-protokube.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ definition: |
Documentation=https://kops.sigs.k8s.io

[Service]
ExecStart=/opt/kops/bin/protokube --bootstrap-master-node-labels=true --cloud=aws --cluster-id=minimal.example.com --containerized=false --dns-internal-suffix=.internal.minimal.example.com --master=true --node-name=master.hostname.invalid --v=4 --zone=*/Z1AFAKE1ZON3YO
ExecStart=/opt/kops/bin/protokube --cloud=aws --containerized=false --master=true --node-name=master.hostname.invalid --v=4
EnvironmentFile=/etc/sysconfig/protokube
Restart=always
RestartSec=3s
Expand Down
40 changes: 6 additions & 34 deletions protokube/cmd/protokube/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,36 +58,24 @@ func main() {

// run is responsible for running the protokube service controller
func run() error {
var zones []string
var containerized, master, gossip bool
var cloud, clusterID, dnsInternalSuffix, gossipSecret, gossipListen, gossipProtocol, gossipSecretSecondary, gossipListenSecondary, gossipProtocolSecondary string
var cloud, gossipSecret, gossipListen, gossipProtocol, gossipSecretSecondary, gossipListenSecondary, gossipProtocolSecondary string
var flagChannels string
var dnsUpdateInterval int

flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized")
flag.BoolVar(&gossip, "gossip", gossip, "Set if we are using gossip dns")
flag.BoolVar(&master, "master", master, "Whether or not this node is a master")
flag.StringVar(&cloud, "cloud", "aws", "CloudProvider we are using (aws,digitalocean,gce,openstack)")
flag.StringVar(&clusterID, "cluster-id", clusterID, "Cluster ID for internal domain names")
flag.StringVar(&dnsInternalSuffix, "dns-internal-suffix", dnsInternalSuffix, "DNS suffix for internal domain names")
flags.IntVar(&dnsUpdateInterval, "dns-update-interval", 5, "Configure interval at which to update DNS records.")
flag.StringVar(&flagChannels, "channels", flagChannels, "channels to install")
flag.StringVar(&gossipProtocol, "gossip-protocol", "mesh", "mesh/memberlist")
flag.StringVar(&gossipListen, "gossip-listen", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipWeaveMesh), "address:port on which to bind for gossip")
flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip")
flag.StringVar(&gossipProtocolSecondary, "gossip-protocol-secondary", "memberlist", "mesh/memberlist")
flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipMemberlist), "address:port on which to bind for gossip")
flags.StringVar(&gossipSecretSecondary, "gossip-secret-secondary", gossipSecret, "Secret to use to secure gossip")
flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")

bootstrapMasterNodeLabels := false
flag.BoolVar(&bootstrapMasterNodeLabels, "bootstrap-master-node-labels", bootstrapMasterNodeLabels, "Bootstrap the labels for master nodes (required in k8s 1.16)")

nodeName := ""
flag.StringVar(&nodeName, "node-name", nodeName, "name of the node as will be created in kubernetes; used with bootstrap-master-node-labels")

var removeDNSNames string
flag.StringVar(&removeDNSNames, "remove-dns-names", removeDNSNames, "If set, will remove the DNS records specified")
flag.StringVar(&nodeName, "node-name", nodeName, "name of the node as will be created in kubernetes")

// Trick to avoid 'logging before flag.Parse' warning
flag.CommandLine.Parse([]string{})
Expand All @@ -96,24 +84,10 @@ func run() error {
flags.AddGoFlagSet(flag.CommandLine)
flags.Parse(os.Args)

if dnsInternalSuffix == "" {
if clusterID == "" {
return fmt.Errorf("cluster-id is required when dns-internal-suffix is not set")
}
// TODO: Maybe only master needs DNS?
dnsInternalSuffix = ".internal." + clusterID
klog.Infof("Setting dns-internal-suffix to %q", dnsInternalSuffix)
}
// Make sure it's actually a suffix (starts with .)
if !strings.HasPrefix(dnsInternalSuffix, ".") {
dnsInternalSuffix = "." + dnsInternalSuffix
}

rootfs := "/"
if containerized {
rootfs = "/rootfs/"
}
protokube.RootFS = rootfs

if gossip {
var cloudProvider protokube.CloudProvider
Expand Down Expand Up @@ -206,12 +180,10 @@ func run() error {
}

k := &protokube.KubeBoot{
BootstrapMasterNodeLabels: bootstrapMasterNodeLabels,
NodeName: nodeName,
Channels: channels,
InternalDNSSuffix: dnsInternalSuffix,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
NodeName: nodeName,
Channels: channels,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
}

k.RunSyncLoop()
Expand Down
23 changes: 4 additions & 19 deletions protokube/pkg/protokube/aws_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,44 +35,29 @@ import (
// AWSCloudProvider defines the AWS cloud provider implementation
type AWSCloudProvider struct {
clusterTag string
deviceMap map[string]string
ec2 ec2.DescribeInstancesAPIClient
instanceId string
imdsClient *imds.Client
zone string
}

var _ CloudProvider = &AWSCloudProvider{}

// NewAWSCloudProvider returns a new aws volume provider
func NewAWSCloudProvider() (*AWSCloudProvider, error) {
ctx := context.TODO()
a := &AWSCloudProvider{
deviceMap: make(map[string]string),
}
a := &AWSCloudProvider{}

config, err := awsconfig.LoadDefaultConfig(ctx, awslog.WithAWSLogger())
if err != nil {
return nil, fmt.Errorf("error loading AWS config: %w", err)
}
a.imdsClient = imds.NewFromConfig(config)
imdsClient := imds.NewFromConfig(config)

regionResp, err := a.imdsClient.GetRegion(ctx, &imds.GetRegionInput{})
regionResp, err := imdsClient.GetRegion(ctx, &imds.GetRegionInput{})
if err != nil {
return nil, fmt.Errorf("error querying ec2 metadata service (for az/region): %w", err)
}

zoneResp, err := a.imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: "placement/availability-zone"})
if err != nil {
return nil, fmt.Errorf("error querying ec2 metadata service (for az): %w", err)
}
zone, err := io.ReadAll(zoneResp.Content)
if err != nil {
return nil, fmt.Errorf("error reading ec2 metadata service response (for az): %w", err)
}
a.zone = string(zone)

instanceIdResp, err := a.imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: "instance-id"})
instanceIdResp, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: "instance-id"})
if err != nil {
return nil, fmt.Errorf("error querying ec2 metadata service (for instance-id): %w", err)
}
Expand Down
14 changes: 1 addition & 13 deletions protokube/pkg/protokube/azure_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,16 @@ limitations under the License.
package protokube

import (
"context"
"fmt"

compute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"k8s.io/kops/protokube/pkg/gossip"
gossipazure "k8s.io/kops/protokube/pkg/gossip/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
)

type client interface {
ListVMScaleSets(ctx context.Context) ([]*compute.VirtualMachineScaleSet, error)
ListVMSSNetworkInterfaces(ctx context.Context, vmScaleSetName string) ([]*network.Interface, error)
GetName() string
GetTags() (map[string]string, error)
}

var _ client = &gossipazure.Client{}

// AzureCloudProvider implements the CloudProvider interface for Azure.
type AzureCloudProvider struct {
client client
client *gossipazure.Client

clusterTag string
instanceID string
Expand Down
56 changes: 4 additions & 52 deletions protokube/pkg/protokube/do_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"io"
"net/http"
"os"
"strconv"
"strings"

"github.com/digitalocean/godo"
Expand All @@ -34,24 +33,15 @@ import (
)

const (
dropletRegionMetadataURL = "http://169.254.169.254/metadata/v1/region"
dropletNameMetadataURL = "http://169.254.169.254/metadata/v1/hostname"
dropletIDMetadataURL = "http://169.254.169.254/metadata/v1/id"
dropletIDMetadataTags = "http://169.254.169.254/metadata/v1/tags"
dropletNameMetadataURL = "http://169.254.169.254/metadata/v1/hostname"
dropletIDMetadataTags = "http://169.254.169.254/metadata/v1/tags"
)

// TokenSource implements oauth2.TokenSource
type TokenSource struct {
AccessToken string
}

type DOCloudProvider struct {
ClusterID string
godoClient *godo.Client

region string
dropletName string
dropletID int
dropletTags []string
}

Expand Down Expand Up @@ -84,20 +74,6 @@ func GetClusterID() (string, error) {
}

func NewDOCloudProvider() (*DOCloudProvider, error) {
region, err := getMetadataRegion()
if err != nil {
return nil, fmt.Errorf("failed to get droplet region: %s", err)
}

dropletIDStr, err := getMetadataDropletID()
if err != nil {
return nil, fmt.Errorf("failed to get droplet id: %s", err)
}
dropletID, err := strconv.Atoi(dropletIDStr)
if err != nil {
return nil, fmt.Errorf("failed to convert droplet ID to int: %s", err)
}

dropletName, err := getMetadataDropletName()
if err != nil {
return nil, fmt.Errorf("failed to get droplet name: %s", err)
Expand All @@ -121,35 +97,19 @@ func NewDOCloudProvider() (*DOCloudProvider, error) {
return &DOCloudProvider{
godoClient: godoClient,
ClusterID: clusterID,
dropletID: dropletID,
dropletName: dropletName,
region: region,
dropletTags: dropletTags,
}, nil
}

// Token() returns oauth2.Token
func (t *TokenSource) Token() (*oauth2.Token, error) {
token := &oauth2.Token{
AccessToken: t.AccessToken,
}
return token, nil
}

func NewDOCloud() (*godo.Client, error) {
accessToken := os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
if accessToken == "" {
return nil, errors.New("DIGITALOCEAN_ACCESS_TOKEN is required")
}

tokenSource := &TokenSource{
AccessToken: accessToken,
}

oauthClient := oauth2.NewClient(context.TODO(), tokenSource)
client := godo.NewClient(oauthClient)

return client, nil
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: accessToken})
return godo.NewClient(oauth2.NewClient(context.TODO(), tokenSource)), nil
}

func (d *DOCloudProvider) GossipSeeds() (gossip.SeedProvider, error) {
Expand All @@ -166,18 +126,10 @@ func (d *DOCloudProvider) InstanceID() string {
return d.dropletName
}

func getMetadataRegion() (string, error) {
return getMetadata(dropletRegionMetadataURL)
}

func getMetadataDropletName() (string, error) {
return getMetadata(dropletNameMetadataURL)
}

func getMetadataDropletID() (string, error) {
return getMetadata(dropletIDMetadataURL)
}

func getMetadataDropletTags() ([]string, error) {
tagString, err := getMetadata(dropletIDMetadataTags)
return strings.Split(tagString, "\n"), err
Expand Down
Loading
Loading