Your advanced code editor is loading.

Please wait a moment.

#editor-loading { display: none; }

// SPDX-License-Identifier: Apache-2.0 // Copyright Authors of Cilium

package linuxrouting

import ( “errors” “fmt” “net” “net/netip”

"github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sys/unix"

"github.com/cilium/cilium/pkg/datapath/linux/linux_defaults" "github.com/cilium/cilium/pkg/datapath/linux/route" iputil "github.com/cilium/cilium/pkg/ip" ipamOption "github.com/cilium/cilium/pkg/ipam/option" "github.com/cilium/cilium/pkg/logging" "github.com/cilium/cilium/pkg/logging/logfields" "github.com/cilium/cilium/pkg/mac" "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option"

)

var ( log = logging.DefaultLogger.WithField(logfields.LogSubsys, “linux-routing”) )

// Configure sets up the rules and routes needed when running in ENI or // Azure IPAM mode. // These rules and routes direct egress traffic out of the interface and // ingress traffic back to the endpoint (ip). The compat flag controls which // egress priority to consider when deleting the egress rules (see // option.Config.EgressMultiHomeIPRuleCompat). // // ip: The endpoint IP address to direct traffic out / from interface. // info: The interface routing info used to create rules and routes. // mtu: The interface MTU. // compat: Whether to use the compat egress priority or not. // host: Whether the IP is a host IP and needs to be routed via the ‘local’ table func (info *RoutingInfo) Configure(ip net.IP, mtu int, compat bool, host bool) error { if ip.To4() == nil { log.WithFields(logrus.Fields{ “endpointIP”: ip, }).Warning(“Unable to configure rules and routes because IP is not an IPv4 address”) return errors.New(“IP not compatible”) }

ifindex, err := retrieveIfIndexFromMAC(info.MasterIfMAC, mtu)
          if err != nil {
          return fmt.Errorf("unable to find ifindex for interface MAC: %w", err)
          }
          
          ipWithMask := net.IPNet{
          IP:   ip,
          Mask: net.CIDRMask(32, 32),
          }
          // Ingress rule. This rule is not installed for the cilium_host IP, because
          // the cilium_host IP is a local IP and therefore must be routed via the
          // 'local' table instead of 'main'.
          if !host {
          // On ingress, route all traffic to the endpoint IP via the main routing
          // table. Egress rules are created in a per-ENI routing table.
          if err := route.ReplaceRule(route.Rule{
          Priority: linux_defaults.RulePriorityIngress,
          To:       &ipWithMask,
          Table:    route.MainTable,
          Protocol: linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to install ip rule: %w", err)
          }
          }
          var egressPriority, tableID int
          if compat {
          egressPriority = linux_defaults.RulePriorityEgress
          tableID = ifindex
          } else {
          egressPriority = linux_defaults.RulePriorityEgressv2
          tableID = computeTableIDFromIfaceNumber(info.InterfaceNumber)
          }
          // The condition here should mirror the condition in Delete.
          if info.Masquerade && info.IpamMode == ipamOption.IPAMENI {
          // Lookup a VPC specific table for all traffic from an endpoint to the
          // CIDR configured for the VPC on which the endpoint has the IP on.
          for _, cidr := range info.IPv4CIDRs {
          if err := route.ReplaceRule(route.Rule{
          Priority: egressPriority,
          From:     &ipWithMask,
          To:       &cidr,
          Table:    tableID,
          Protocol: linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to install ip rule: %w", err)
          }
          }
          } else {
          // Lookup a VPC specific table for all traffic from an endpoint.
          if err := route.ReplaceRule(route.Rule{
          Priority: egressPriority,
          From:     &ipWithMask,
          Table:    tableID,
          Protocol: linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to install ip rule: %w", err)
          }
          }
          // Nexthop route to the VPC or subnet gateway
          //
          // Note: This is a /32 route to avoid any L2. The endpoint does no L2
          // either.
          if err := netlink.RouteReplace(&netlink.Route{
          LinkIndex: ifindex,
          Dst:       &net.IPNet{IP: info.IPv4Gateway, Mask: net.CIDRMask(32, 32)},
          Scope:     netlink.SCOPE_LINK,
          Table:     tableID,
          Protocol:  linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to add L2 nexthop route: %w", err)
          }
          // Default route to the VPC or subnet gateway
          if err := netlink.RouteReplace(&netlink.Route{
          Dst:      &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
          Table:    tableID,
          Gw:       info.IPv4Gateway,
          Protocol: linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to add L2 nexthop route: %w", err)
          }
          return nil

}

// Delete removes the ingress and egress rules that control traffic for // endpoints. Note that the routes referenced by the rules are not deleted as // they can be reused when another endpoint is created on the same node. The // compat flag controls which egress priority to consider when deleting the // egress rules (see option.Config.EgressMultiHomeIPRuleCompat). // // Note that one or more IPs may share the same route table, as identified by // the interface number of the corresponding device. This function only removes // the ingress and egress rules to disconnect the per-ENI egress routes from a // specific local IP, and does not remove the corresponding route table as // other IPs may still be using that table. // // The search for both the ingress & egress rule corresponding to this IP is a // best-effort based on the respective priority that Cilium uses, which we // assume full control over. The search for the ingress rule is more likely to // succeed (albeit very rarely that egress deletion fails) because we are able // to perform a narrower search on the rule because we know it references the // main routing table. Due to multiple routing CIDRs, there might be more than // one egress rule. Deletion of any rule only proceeds if the rule matches // the IP & priority. If more than one rule matches, then deletion is skipped. func Delete(ip netip.Addr, compat bool) error { if !ip.Is4() { log.WithFields(logrus.Fields{ “endpointIP”: ip, }).Warning(“Unable to delete rules because IP is not an IPv4 address”) return errors.New(“IP not compatible”) }

ipWithMask := iputil.AddrToIPNet(ip)
          
          scopedLog := log.WithFields(logrus.Fields{
          "ip": ipWithMask.String(),
          })
          // Ingress rules
          ingress := route.Rule{
          Priority: linux_defaults.RulePriorityIngress,
          To:       ipWithMask,
          Table:    route.MainTable,
          }
          if err := deleteRule(ingress); err != nil {
          return fmt.Errorf("unable to delete ingress rule from main table with ip %s: %w", ipWithMask.String(), err)
          }
          scopedLog.WithField("rule", ingress).Debug("Deleted ingress rule")
          priority := linux_defaults.RulePriorityEgressv2
          if compat {
          priority = linux_defaults.RulePriorityEgress
          }
          // Egress rules
          // The condition here should mirror the conditions in Configure.
          info := node.GetRouterInfo()
          if info != nil && option.Config.EnableIPv4Masquerade && option.Config.IPAM == ipamOption.IPAMENI {
          ipv4CIDRs := info.GetIPv4CIDRs()
          cidrs := make([]*net.IPNet, 0, len(ipv4CIDRs))
          for i := range ipv4CIDRs {
          cidrs = append(cidrs, &ipv4CIDRs[i])
          }
          // Coalesce CIDRs into minimum set needed for route rules
          // This code here mirrors interfaceAdd() in cilium-cni/interface.go
          // and must be kept in sync when modified
          routingCIDRs, _ := iputil.CoalesceCIDRs(cidrs)
          for _, cidr := range routingCIDRs {
          egress := route.Rule{
          Priority: priority,
          From:     ipWithMask,
          To:       cidr,
          }
          if err := deleteRule(egress); err != nil {
          return fmt.Errorf("unable to delete egress rule with ip %s: %w", ipWithMask.String(), err)
          }
          scopedLog.WithField(logfields.Rule, egress).Debug("Deleted egress rule")
          }
          } else {
          egress := route.Rule{
          Priority: priority,
          From:     ipWithMask,
          }
          if err := deleteRule(egress); err != nil {
          return fmt.Errorf("unable to delete egress rule with ip %s: %w", ipWithMask.String(), err)
          }
          scopedLog.WithField(logfields.Rule, egress).Debug("Deleted egress rule")
          }
          if option.Config.EnableUnreachableRoutes {
          // Replace route to old IP with an unreachable route. This will
          //   - trigger ICMP error messages for clients attempting to connect to the stale IP
          //   - avoid hitting rp_filter and getting Martian packet warning
          // When the IP is reused, the unreachable route will be replaced to target the new pod veth
          // In CRD-based IPAM, when an IP is unassigned from the CiliumNode, we delete this route
          // to avoid blackholing traffic to this IP if it gets reassigned to another node
          if err := netlink.RouteReplace(&netlink.Route{
          Dst:      ipWithMask,
          Table:    route.MainTable,
          Type:     unix.RTN_UNREACHABLE,
          Protocol: linux_defaults.RTProto,
          }); err != nil {
          return fmt.Errorf("unable to add unreachable route for ip %s: %w", ipWithMask.String(), err)
          }
          }
          return nil

}

func deleteRule(r route.Rule) error { rules, err := route.ListRules(netlink.FAMILY_V4, &r) if err != nil { return err }

length := len(rules)
          switch {
          case length > 1:
          log.WithFields(logrus.Fields{
          "candidates": rules,
          "rule":       r,
          }).Warning("Found too many rules matching, skipping deletion")
          return errors.New("unexpected number of rules found to delete")
          case length == 1:
          return route.DeleteRule(netlink.FAMILY_V4, r)
          }
          
          log.WithFields(logrus.Fields{
          "rule": r,
          }).Warning("No rule matching found")
          return errors.New("no rule found to delete")

}

// retrieveIfIndexFromMAC finds the corresponding device index (ifindex) for a // given MAC address, excluding Linux slave devices. This is useful for // creating rules and routes in order to specify the table. When the ifindex is // found, the device is brought up and its MTU is set. func retrieveIfIndexFromMAC(mac mac.MAC, mtu int) (int, error) { var link netlink.Link

links, err := netlink.LinkList()
          if err != nil {
          return -1, fmt.Errorf("unable to list interfaces: %w", err)
          }
          
          for _, l := range links {
          // Linux slave devices have the same MAC address as their master
          // device, but we want the master device.
          if l.Attrs().RawFlags&unix.IFF_SLAVE != 0 {
          continue
          }
          if l.Attrs().HardwareAddr.String() == mac.String() {
          if link != nil {
          return -1, fmt.Errorf("several interfaces found with MAC %s: %s and %s", mac, link.Attrs().Name, l.Attrs().Name)
          }
          link = l
          }
          }
          if link == nil {
          return -1, fmt.Errorf("interface with MAC %s not found", mac)
          }
          if err = netlink.LinkSetMTU(link, mtu); err != nil {
          return -1, fmt.Errorf("unable to change MTU of link %s to %d: %w", link.Attrs().Name, mtu, err)
          }
          if err = netlink.LinkSetUp(link); err != nil {
          return -1, fmt.Errorf("unable to up link %s: %w", link.Attrs().Name, err)
          }
          return link.Attrs().Index, nil

}

// computeTableIDFromIfaceNumber returns a computed per-ENI route table ID for the given // ENI interface number. func computeTableIDFromIfaceNumber(num int) int { return linux_defaults.RouteTableInterfacesOffset + num }

Explanation

This code in routing.go from the Cilium repository deals with configuring and deleting routing rules and routes for endpoints in an ENI or Azure IPAM environment. It’s primarily concerned with managing egress and ingress traffic for endpoints, directing it through the correct interfaces and routes.

Key Concepts:

ENI (Elastic Network Interface): A virtual network interface in AWS, used for connecting instances to a VPC.
IPAM (IP Address Management): A system for managing IP addresses within a network.
Routing Table: A table that maps network destinations to the next hop for sending packets.
Rule: A rule in the routing table that determines how packets are handled based on their source, destination, and other criteria.
Nexthop: The next hop router or gateway that a packet should be sent to.
Masquerade: A mechanism where the source IP of a packet is changed to the IP of the router before sending it out.

Configure Function:

This function configures the routing rules and routes needed for an endpoint. It takes the endpoint’s IP address (ip), the MTU of the interface (mtu), a compatibility flag (compat), and a flag indicating whether the IP is a host IP (host).

Validation: It checks if the provided IP is an IPv4 address.
Interface Index: It retrieves the interface index (ifindex) for the interface corresponding to the provided MAC address (info.MasterIfMAC).
Ingress Rule: If the IP is not a host IP, it creates an ingress rule in the main routing table, directing all traffic to the endpoint’s IP through the interface.
Egress Priority and Table: Determines the egress priority and routing table ID based on the compat flag. If compat is true, it uses the older egress priority (linux_defaults.RulePriorityEgress) and the interface index as the table ID. Otherwise, it uses the newer egress priority (linux_defaults.RulePriorityEgressv2) and a computed table ID based on the interface number.
Egress Rule: Based on the Masquerade flag and the IPAM mode, it creates egress rules in the specified routing table. For ENI IPAM, it creates a rule for each CIDR configured for the VPC. Otherwise, it creates a rule for all traffic from the endpoint.
Nexthop Routes: Creates two routes to the VPC or subnet gateway:

A /32 route to ensure L2 traffic doesn’t get misdirected.
A default route to the gateway for all other destinations.

Delete Function:

This function removes the ingress and egress rules created by Configure for an endpoint. It takes the endpoint’s IP address (ip) and a compatibility flag (compat).

Validation: Checks if the provided IP is an IPv4 address.
Ingress Rule Deletion: Deletes the ingress rule in the main routing table that matches the provided IP.
Egress Rule Deletion: Deletes the egress rules matching the provided IP and priority. It considers the compatibility flag for determining the correct priority.
Unreachable Route (Optional): If option.Config.EnableUnreachableRoutes is enabled, it creates an unreachable route for the old IP to trigger ICMP error messages for clients trying to connect to it. This is meant to prevent blackholing traffic when the IP is reused.

deleteRule Function:

This helper function deletes a routing rule. It takes a route.Rule as input, retrieves rules matching the provided parameters, and deletes the rule if only one match is found. If more than one rule matches, it logs a warning and skips deletion.

Other Functions:

retrieveIfIndexFromMAC: Finds the interface index for a given MAC address, excluding Linux slave devices. It also sets up the interface by setting its MTU and bringing it up.
computeTableIDFromIfaceNumber: Calculates the routing table ID for an ENI based on its interface number.

Overall: This code effectively manages routing rules and routes for endpoints in ENI and Azure IPAM environments, ensuring correct handling of egress and ingress traffic. It’s carefully designed to handle various configurations, including different IPAM modes, masquerading, and compatibility options for egress rule priorities.

Your advanced code editor is loading.

Please wait a moment.

Explanation

Graph

Symbols

We couldn't identify any entrypoints. If you believe this to be incorrect then please contact support.