[INET]: Generalise the TCP sock ID lookup routines

And also some TIME_WAIT functions.

[acme@toy net-2.6.14]$ grep built-in /tmp/before.size /tmp/after.size
/tmp/before.size: 282955   13122    9312  305389   4a8ed net/ipv4/built-in.o
/tmp/after.size:  281566   13122    9312  304000   4a380 net/ipv4/built-in.o
[acme@toy net-2.6.14]$

I kept them still inlined, will uninline at some point to see what
would be the performance difference.

Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>

3 files changed, 87 insertions, 11 deletions

diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index c38c637e073..b5c0d64ea74 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -30,6 +30,7 @@
 #include <net/tcp_states.h>
 
 #include <asm/atomic.h>
+#include <asm/byteorder.h>
 
 /* This is for all connections with a full identity, no wildcards.
  * New scheme, half the table is for TIME_WAIT, the other half is
@@ -285,13 +286,13 @@ extern struct sock *__inet_lookup_listener(const struct hlist_head *head,
 					   const int dif);
 
 /* Optimize the common listener case. */
-static inline struct sock *inet_lookup_listener(struct inet_hashinfo *hashinfo,
-						const u32 daddr,
-						const unsigned short hnum,
-						const int dif)
+static inline struct sock *
+		inet_lookup_listener(struct inet_hashinfo *hashinfo,
+				     const u32 daddr,
+				     const unsigned short hnum, const int dif)
 {
 	struct sock *sk = NULL;
-	struct hlist_head *head;
+	const struct hlist_head *head;
 
 	read_lock(&hashinfo->lhash_lock);
 	head = &hashinfo->listening_hash[inet_lhashfn(hnum)];
@@ -351,4 +352,70 @@ sherry_cache:
 	 ((*((__u32 *)&(inet_twsk(__sk)->tw_dport))) == (__ports)) &&	\
 	 (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
 #endif /* 64-bit arch */
+
+/*
+ * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need
+ * not check it for lookups anymore, thanks Alexey. -DaveM
+ *
+ * Local BH must be disabled here.
+ */
+static inline struct sock *
+	__inet_lookup_established(struct inet_hashinfo *hashinfo,
+				  const u32 saddr, const u16 sport,
+				  const u32 daddr, const u16 hnum,
+				  const int dif)
+{
+	INET_ADDR_COOKIE(acookie, saddr, daddr)
+	const __u32 ports = INET_COMBINED_PORTS(sport, hnum);
+	struct sock *sk;
+	const struct hlist_node *node;
+	/* Optimize here for direct hit, only listening connections can
+	 * have wildcards anyways.
+	 */
+	const int hash = inet_ehashfn(daddr, hnum, saddr, sport, hashinfo->ehash_size);
+	struct inet_ehash_bucket *head = &hashinfo->ehash[hash];
+
+	read_lock(&head->lock);
+	sk_for_each(sk, node, &head->chain) {
+		if (INET_MATCH(sk, acookie, saddr, daddr, ports, dif))
+			goto hit; /* You sunk my battleship! */
+	}
+
+	/* Must check for a TIME_WAIT'er before going to listener hash. */
+	sk_for_each(sk, node, &(head + hashinfo->ehash_size)->chain) {
+		if (INET_TW_MATCH(sk, acookie, saddr, daddr, ports, dif))
+			goto hit;
+	}
+	sk = NULL;
+out:
+	read_unlock(&head->lock);
+	return sk;
+hit:
+	sock_hold(sk);
+	goto out;
+}
+
+static inline struct sock *__inet_lookup(struct inet_hashinfo *hashinfo,
+					 const u32 saddr, const u16 sport,
+					 const u32 daddr, const u16 hnum,
+					 const int dif)
+{
+	struct sock *sk = __inet_lookup_established(hashinfo, saddr, sport, daddr,
+						    hnum, dif);
+	return sk ? : inet_lookup_listener(hashinfo, daddr, hnum, dif);
+}
+
+static inline struct sock *inet_lookup(struct inet_hashinfo *hashinfo,
+				       const u32 saddr, const u16 sport,
+				       const u32 daddr, const u16 dport,
+				       const int dif)
+{
+	struct sock *sk;
+
+	local_bh_disable();
+	sk = __inet_lookup(hashinfo, saddr, sport, daddr, ntohs(dport), dif);
+	local_bh_enable();
+
+	return sk;
+}
 #endif /* _INET_HASHTABLES_H */
diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index ce117048f2f..020f28058fd 100644
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -17,6 +17,7 @@
 
 #include <linux/config.h>
 
+#include <linux/ip.h>
 #include <linux/list.h>
 #include <linux/types.h>
 
@@ -32,6 +33,7 @@
 #endif
 
 struct inet_bind_bucket;
+struct inet_hashinfo;
 
 /*
  * This is a TIME_WAIT sock. It works around the memory consumption
@@ -139,4 +141,11 @@ static inline void inet_twsk_put(struct inet_timewait_sock *tw)
 		kmem_cache_free(tw->tw_prot->twsk_slab, tw);
 	}
 }
+
+extern void __inet_twsk_kill(struct inet_timewait_sock *tw,
+			     struct inet_hashinfo *hashinfo);
+
+extern void __inet_twsk_hashdance(struct inet_timewait_sock *tw,
+				  struct sock *sk,
+				  struct inet_hashinfo *hashinfo);
 #endif	/* _INET_TIMEWAIT_SOCK_ */
diff --git a/include/net/sock.h b/include/net/sock.h
index c902c57bf2b..bdae0a5eadf 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -255,28 +255,28 @@ struct sock {
 /*
  * Hashed lists helper routines
  */
-static inline struct sock *__sk_head(struct hlist_head *head)
+static inline struct sock *__sk_head(const struct hlist_head *head)
 {
 	return hlist_entry(head->first, struct sock, sk_node);
 }
 
-static inline struct sock *sk_head(struct hlist_head *head)
+static inline struct sock *sk_head(const struct hlist_head *head)
 {
 	return hlist_empty(head) ? NULL : __sk_head(head);
 }
 
-static inline struct sock *sk_next(struct sock *sk)
+static inline struct sock *sk_next(const struct sock *sk)
 {
 	return sk->sk_node.next ?
 		hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
 }
 
-static inline int sk_unhashed(struct sock *sk)
+static inline int sk_unhashed(const struct sock *sk)
 {
 	return hlist_unhashed(&sk->sk_node);
 }
 
-static inline int sk_hashed(struct sock *sk)
+static inline int sk_hashed(const struct sock *sk)
 {
 	return sk->sk_node.pprev != NULL;
 }
@@ -494,7 +494,7 @@ extern int sk_wait_data(struct sock *sk, long *timeo);
 struct request_sock_ops;
 
 /* Here is the right place to enable sock refcounting debugging */
-#define SOCK_REFCNT_DEBUG
+//#define SOCK_REFCNT_DEBUG
 
 /* Networking protocol blocks we attach to sockets.
  * socket layer -> transport layer interface