am 798f9934: Use different IPv4 addresses on different clat interfaces.

* commit '798f9934fca523dfb57136bd185cf6e9460323ad':
  Use different IPv4 addresses on different clat interfaces.

5 files changed, 154 insertions, 5 deletions

diff --git a/clatd.c b/clatd.c
index 41e961e..3f0af0b 100644
--- a/clatd.c
+++ b/clatd.c
@@ -157,6 +157,22 @@ int configure_packet_socket(int sock) {
 void configure_tun_ip(const struct tun_data *tunnel) {
   int status;
 
+  // Pick an IPv4 address to use by finding a free address in the configured prefix. Technically,
+  // there is a race here - if another clatd calls config_select_ipv4_address after we do, but
+  // before we call add_address, it can end up having the same IP address as we do. But the time
+  // window in which this can happen is extremely small, and even if we end up with a duplicate
+  // address, the only damage is that IPv4 TCP connections won't be reset until both interfaces go
+  // down.
+  in_addr_t localaddr = config_select_ipv4_address(&Global_Clatd_Config.ipv4_local_subnet,
+                                                   Global_Clatd_Config.ipv4_local_prefixlen);
+  if (localaddr == INADDR_NONE) {
+    logmsg(ANDROID_LOG_FATAL,"No free IPv4 address in %s/%d",
+           inet_ntoa(Global_Clatd_Config.ipv4_local_subnet),
+           Global_Clatd_Config.ipv4_local_prefixlen);
+    exit(1);
+  }
+  Global_Clatd_Config.ipv4_local_subnet.s_addr = localaddr;
+
   // Configure the interface before bringing it up. As soon as we bring the interface up, the
   // framework will be notified and will assume the interface's configuration has been finalized.
   status = add_address(tunnel->device4, AF_INET, &Global_Clatd_Config.ipv4_local_subnet,
@@ -166,6 +182,10 @@ void configure_tun_ip(const struct tun_data *tunnel) {
     exit(1);
   }
 
+  char addrstr[INET_ADDRSTRLEN];
+  inet_ntop(AF_INET, &Global_Clatd_Config.ipv4_local_subnet, addrstr, sizeof(addrstr));
+  logmsg(ANDROID_LOG_INFO, "Using IPv4 address %s on %s", addrstr, tunnel->device4);
+
   if((status = if_up(tunnel->device4, Global_Clatd_Config.ipv4mtu)) < 0) {
     logmsg(ANDROID_LOG_FATAL,"configure_tun_ip/if_up(4) failed: %s",strerror(-status));
     exit(1);
@@ -269,7 +289,7 @@ int update_clat_ipv6_address(const struct tun_data *tunnel, const char *interfac
 
   if (IN6_IS_ADDR_UNSPECIFIED(&Global_Clatd_Config.ipv6_local_subnet)) {
     // Startup.
-    logmsg(ANDROID_LOG_INFO, "Using %s on %s", addrstr, interface);
+    logmsg(ANDROID_LOG_INFO, "Using IPv6 address %s on %s", addrstr, interface);
   } else {
     // Prefix change.
     char from_addr[INET6_ADDRSTRLEN];
diff --git a/clatd.conf b/clatd.conf
index 3805c6d..ff80975 100644
--- a/clatd.conf
+++ b/clatd.conf
@@ -3,8 +3,14 @@
 # A host IID of :: means to generate a checksum-neutral, random IID.
 ipv6_host_id ::
 
-# ipv4 subnet for the local traffic to use.  This is a /32 host address
+# IPv4 address configuration to use when selecting a host address. The first
+# clat daemon started will use the address specified by ipv4_local_subnet. If
+# more than one daemon is run at the same time, subsequent daemons will use
+# other addresses in the prefix of length ipv4_local prefixlen that contains
+# ipv4_local_subnet. The default is to use the IANA-assigned range 192.0.0.0/29,
+# which allows up to 8 clat daemons (.4, .5, .6, .7, .0, .1, .2, .3).
 ipv4_local_subnet 192.0.0.4
+ipv4_local_prefixlen 29
 
 # get the plat_subnet from dns lookups (requires DNS64)
 plat_from_dns64 yes
diff --git a/clatd_test.cpp b/clatd_test.cpp
index 50d9677..fd429ca 100644
--- a/clatd_test.cpp
+++ b/clatd_test.cpp
@@ -683,6 +683,59 @@ TEST_F(ClatdTest, TestGenIIDRandom) {
   EXPECT_GE(3210000, onebits);
 }
 
+extern "C" addr_free_func config_is_ipv4_address_free;
+int never_free(in_addr_t /* addr */) { return 0; }
+int always_free(in_addr_t /* addr */) { return 1; }
+int only2_free(in_addr_t addr) { return (ntohl(addr) & 0xff) == 2; }
+int over6_free(in_addr_t addr) { return (ntohl(addr) & 0xff) >= 6; }
+int only10_free(in_addr_t addr) { return (ntohl(addr) & 0xff) == 10; }
+
+TEST_F(ClatdTest, SelectIPv4Address) {
+  struct in_addr addr;
+
+  inet_pton(AF_INET, kIPv4LocalAddr, &addr);
+
+  addr_free_func orig_config_is_ipv4_address_free = config_is_ipv4_address_free;
+
+  // If no addresses are free, return INADDR_NONE.
+  config_is_ipv4_address_free = never_free;
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 29));
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 16));
+
+  // If the configured address is free, pick that. But a prefix that's too big is invalid.
+  config_is_ipv4_address_free = always_free;
+  EXPECT_EQ(inet_addr(kIPv4LocalAddr), config_select_ipv4_address(&addr, 29));
+  EXPECT_EQ(inet_addr(kIPv4LocalAddr), config_select_ipv4_address(&addr, 20));
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 15));
+
+  // A prefix length of 32 works, but anything above it is invalid.
+  EXPECT_EQ(inet_addr(kIPv4LocalAddr), config_select_ipv4_address(&addr, 32));
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 33));
+
+  // If another address is free, pick it.
+  config_is_ipv4_address_free = over6_free;
+  EXPECT_EQ(inet_addr("192.0.0.6"), config_select_ipv4_address(&addr, 29));
+
+  // Check that we wrap around to addresses that are lower than the first address.
+  config_is_ipv4_address_free = only2_free;
+  EXPECT_EQ(inet_addr("192.0.0.2"), config_select_ipv4_address(&addr, 29));
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 30));
+
+  // If a free address exists outside the prefix, we don't pick it.
+  config_is_ipv4_address_free = only10_free;
+  EXPECT_EQ(INADDR_NONE, config_select_ipv4_address(&addr, 29));
+  EXPECT_EQ(inet_addr("192.0.0.10"), config_select_ipv4_address(&addr, 24));
+
+  // Now try using the real function which sees if IP addresses are free using bind().
+  // Assume that the machine running the test has the address 127.0.0.1, but not 8.8.8.8.
+  config_is_ipv4_address_free = orig_config_is_ipv4_address_free;
+  addr.s_addr = inet_addr("8.8.8.8");
+  EXPECT_EQ(inet_addr("8.8.8.8"), config_select_ipv4_address(&addr, 29));
+
+  addr.s_addr = inet_addr("127.0.0.1");
+  EXPECT_EQ(inet_addr("127.0.0.2"), config_select_ipv4_address(&addr, 29));
+}
+
 TEST_F(ClatdTest, DataSanitycheck) {
   // Sanity checks the data.
   uint8_t v4_header[] = { IPV4_UDP_HEADER };
diff --git a/config.c b/config.c
index 4939478..de8a26f 100644
--- a/config.c
+++ b/config.c
@@ -185,7 +185,12 @@ void dns64_detection(unsigned net_id) {
   }
 }
 
-
+/* function: gen_random_iid
+ * picks a random interface ID that is checksum neutral with the IPv4 address and the NAT64 prefix
+ * myaddr            - IPv6 address to write to
+ * ipv4_local_subnet - clat IPv4 address
+ * plat_subnet       - NAT64 prefix
+ */
 void gen_random_iid(struct in6_addr *myaddr, struct in_addr *ipv4_local_subnet,
                     struct in6_addr *plat_subnet) {
   // Fill last 8 bytes of IPv6 address with random bits.
@@ -208,6 +213,61 @@ void gen_random_iid(struct in6_addr *myaddr, struct in_addr *ipv4_local_subnet,
   myaddr->s6_addr[12] = delta & 0xff;
 }
 
+// Factored out to a separate function for testability.
+int connect_is_ipv4_address_free(in_addr_t addr) {
+  int s = socket(AF_INET, SOCK_DGRAM, 0);
+  if (s == -1) {
+    return 0;
+  }
+
+  // Attempt to connect to the address. If the connection succeeds and getsockname returns the same
+  // the address then the address is already assigned to the system and we can't use it.
+  struct sockaddr_in sin = { .sin_family = AF_INET, .sin_addr = { addr }, .sin_port = 53 };
+  socklen_t len = sizeof(sin);
+  int inuse = connect(s, (struct sockaddr *) &sin, sizeof(sin)) == 0 &&
+              getsockname(s, (struct sockaddr *) &sin, &len) == 0 &&
+              (size_t) len >= sizeof(sin) &&
+              sin.sin_addr.s_addr == addr;
+
+  close(s);
+  return !inuse;
+}
+
+addr_free_func config_is_ipv4_address_free = connect_is_ipv4_address_free;
+
+/* function: config_select_ipv4_address
+ * picks a free IPv4 address, starting from ip and trying all addresses in the prefix in order
+ * ip        - the IP address from the configuration file
+ * prefixlen - the length of the prefix from which addresses may be selected.
+ * returns: the IPv4 address, or INADDR_NONE if no addresses were available
+ */
+in_addr_t config_select_ipv4_address(const struct in_addr *ip, int16_t prefixlen) {
+  in_addr_t chosen = INADDR_NONE;
+
+  // Don't accept prefixes that are too large because we scan addresses one by one.
+  if (prefixlen < 16 || prefixlen > 32) {
+      return chosen;
+  }
+
+  // All these are in host byte order.
+  in_addr_t mask = 0xffffffff >> (32 - prefixlen) << (32 - prefixlen);
+  in_addr_t ipv4 = ntohl(ip->s_addr);
+  in_addr_t first_ipv4 = ipv4;
+  in_addr_t prefix = ipv4 & mask;
+
+  // Pick the first IPv4 address in the pool, wrapping around if necessary.
+  // So, for example, 192.0.0.4 -> 192.0.0.5 -> 192.0.0.6 -> 192.0.0.7 -> 192.0.0.0.
+  do {
+     if (config_is_ipv4_address_free(htonl(ipv4))) {
+       chosen = htonl(ipv4);
+       break;
+     }
+     ipv4 = prefix | ((ipv4 + 1) & ~mask);
+  } while (ipv4 != first_ipv4);
+
+  return chosen;
+}
+
 /* function: config_generate_local_ipv6_subnet
  * generates the local ipv6 subnet when given the interface ip
  * requires config.ipv6_host_id
@@ -264,7 +324,12 @@ int read_config(const char *file, const char *uplink_interface, const char *plat
   if(!config_item_int16_t(root, "ipv4mtu", "-1", &Global_Clatd_Config.ipv4mtu))
     goto failed;
 
-  if(!config_item_ip(root, "ipv4_local_subnet", DEFAULT_IPV4_LOCAL_SUBNET, &Global_Clatd_Config.ipv4_local_subnet))
+  if(!config_item_ip(root, "ipv4_local_subnet", DEFAULT_IPV4_LOCAL_SUBNET,
+                     &Global_Clatd_Config.ipv4_local_subnet))
+    goto failed;
+
+  if(!config_item_int16_t(root, "ipv4_local_prefixlen", DEFAULT_IPV4_LOCAL_PREFIXLEN,
+                          &Global_Clatd_Config.ipv4_local_prefixlen))
     goto failed;
 
   if(plat_prefix) { // plat subnet is coming from the command line
@@ -311,6 +376,7 @@ void dump_config() {
   logmsg(ANDROID_LOG_DEBUG,"ipv4mtu = %d",Global_Clatd_Config.ipv4mtu);
   logmsg(ANDROID_LOG_DEBUG,"ipv6_local_subnet = %s",inet_ntop(AF_INET6, &Global_Clatd_Config.ipv6_local_subnet, charbuffer, sizeof(charbuffer)));
   logmsg(ANDROID_LOG_DEBUG,"ipv4_local_subnet = %s",inet_ntop(AF_INET, &Global_Clatd_Config.ipv4_local_subnet, charbuffer, sizeof(charbuffer)));
+  logmsg(ANDROID_LOG_DEBUG,"ipv4_local_prefixlen = %d", Global_Clatd_Config.ipv4_local_prefixlen);
   logmsg(ANDROID_LOG_DEBUG,"plat_subnet = %s",inet_ntop(AF_INET6, &Global_Clatd_Config.plat_subnet, charbuffer, sizeof(charbuffer)));
   logmsg(ANDROID_LOG_DEBUG,"default_pdp_interface = %s",Global_Clatd_Config.default_pdp_interface);
 }
diff --git a/config.h b/config.h
index a56d6fc..05b3a5a 100644
--- a/config.h
+++ b/config.h
@@ -21,7 +21,7 @@
 #include <netinet/in.h>
 
 #define DEFAULT_IPV4_LOCAL_SUBNET "192.0.0.4"
-
+#define DEFAULT_IPV4_LOCAL_PREFIXLEN "29"
 #define DEFAULT_DNS64_DETECTION_HOSTNAME "ipv4only.arpa"
 
 struct clat_config {
@@ -29,6 +29,7 @@ struct clat_config {
   struct in6_addr ipv6_local_subnet;
   struct in6_addr ipv6_host_id;
   struct in_addr ipv4_local_subnet;
+  int16_t ipv4_local_prefixlen;
   struct in6_addr plat_subnet;
   char *default_pdp_interface;
   char *plat_from_dns64_hostname;
@@ -39,6 +40,9 @@ extern struct clat_config Global_Clatd_Config;
 int read_config(const char *file, const char *uplink_interface, const char *plat_prefix,
         unsigned net_id);
 void config_generate_local_ipv6_subnet(struct in6_addr *interface_ip);
+in_addr_t config_select_ipv4_address(const struct in_addr *ip, int16_t prefixlen);
 int ipv6_prefix_equal(struct in6_addr *a1, struct in6_addr *a2);
 
+typedef int (*addr_free_func)(in_addr_t addr);
+
 #endif /* __CONFIG_H__ */