[deliverable/linux.git] / net / ipv4 / gre_offload.c

/*
 *	IPV4 GSO/GRO offload support
 *	Linux INET implementation
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 *
 *	GRE GSO support
 */

#include <linux/skbuff.h>
#include <linux/init.h>
#include <net/protocol.h>
#include <net/gre.h>

static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
				       netdev_features_t features)
{
	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
	struct sk_buff *segs = ERR_PTR(-EINVAL);
	u16 mac_offset = skb->mac_header;
	__be16 protocol = skb->protocol;
	u16 mac_len = skb->mac_len;
	int gre_offset, outer_hlen;
	bool need_csum;

	if (unlikely(skb_shinfo(skb)->gso_type &
				~(SKB_GSO_TCPV4 |
				  SKB_GSO_TCPV6 |
				  SKB_GSO_UDP |
				  SKB_GSO_DODGY |
				  SKB_GSO_TCP_ECN |
				  SKB_GSO_GRE |
				  SKB_GSO_GRE_CSUM |
				  SKB_GSO_IPIP |
				  SKB_GSO_SIT)))
		goto out;

	if (!skb->encapsulation)
		goto out;

	if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr)))
		goto out;

	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
		goto out;

	/* setup inner skb. */
	skb->encapsulation = 0;
	__skb_pull(skb, tnl_hlen);
	skb_reset_mac_header(skb);
	skb_set_network_header(skb, skb_inner_network_offset(skb));
	skb->mac_len = skb_inner_network_offset(skb);
	skb->protocol = skb->inner_protocol;

	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM);
	skb->encap_hdr_csum = need_csum;

	features &= skb->dev->hw_enc_features;

	/* segment inner packet. */
	segs = skb_mac_gso_segment(skb, features);
	if (IS_ERR_OR_NULL(segs)) {
		skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
				     mac_len);
		goto out;
	}

	outer_hlen = skb_tnl_header_len(skb);
	gre_offset = outer_hlen - tnl_hlen;
	skb = segs;
	do {
		struct gre_base_hdr *greh;
		__be32 *pcsum;

		skb_reset_inner_headers(skb);
		skb->encapsulation = 1;

		skb->mac_len = mac_len;
		skb->protocol = protocol;

		__skb_push(skb, outer_hlen);
		skb_reset_mac_header(skb);
		skb_set_network_header(skb, mac_len);
		skb_set_transport_header(skb, gre_offset);

		if (!need_csum)
			continue;

		greh = (struct gre_base_hdr *)skb_transport_header(skb);
		pcsum = (__be32 *)(greh + 1);

		*pcsum = 0;
		*(__sum16 *)pcsum = gso_make_checksum(skb, 0);
	} while ((skb = skb->next));
out:
	return segs;
}

static struct sk_buff **gre_gro_receive(struct sk_buff **head,
					struct sk_buff *skb)
{
	struct sk_buff **pp = NULL;
	struct sk_buff *p;
	const struct gre_base_hdr *greh;
	unsigned int hlen, grehlen;
	unsigned int off;
	int flush = 1;
	struct packet_offload *ptype;
	__be16 type;

	off = skb_gro_offset(skb);
	hlen = off + sizeof(*greh);
	greh = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out;
	}

	/* Only support version 0 and K (key), C (csum) flags. Note that
	 * although the support for the S (seq#) flag can be added easily
	 * for GRO, this is problematic for GSO hence can not be enabled
	 * here because a GRO pkt may end up in the forwarding path, thus
	 * requiring GSO support to break it up correctly.
	 */
	if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
		goto out;

	type = greh->protocol;

	rcu_read_lock();
	ptype = gro_find_receive_by_type(type);
	if (!ptype)
		goto out_unlock;

	grehlen = GRE_HEADER_SECTION;

	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	hlen = off + grehlen;
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out_unlock;
	}

	/* Don't bother verifying checksum if we're going to flush anyway. */
	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
		if (skb_gro_checksum_simple_validate(skb))
			goto out_unlock;

		skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0,
					     null_compute_pseudo);
	}

	flush = 0;

	for (p = *head; p; p = p->next) {
		const struct gre_base_hdr *greh2;

		if (!NAPI_GRO_CB(p)->same_flow)
			continue;

		/* The following checks are needed to ensure only pkts
		 * from the same tunnel are considered for aggregation.
		 * The criteria for "the same tunnel" includes:
		 * 1) same version (we only support version 0 here)
		 * 2) same protocol (we only support ETH_P_IP for now)
		 * 3) same set of flags
		 * 4) same key if the key field is present.
		 */
		greh2 = (struct gre_base_hdr *)(p->data + off);

		if (greh2->flags != greh->flags ||
		    greh2->protocol != greh->protocol) {
			NAPI_GRO_CB(p)->same_flow = 0;
			continue;
		}
		if (greh->flags & GRE_KEY) {
			/* compare keys */
			if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
				NAPI_GRO_CB(p)->same_flow = 0;
				continue;
			}
		}
	}

	skb_gro_pull(skb, grehlen);

	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	skb_gro_postpull_rcsum(skb, greh, grehlen);

	pp = ptype->callbacks.gro_receive(head, skb);

out_unlock:
	rcu_read_unlock();
out:
	NAPI_GRO_CB(skb)->flush |= flush;

	return pp;
}

static int gre_gro_complete(struct sk_buff *skb, int nhoff)
{
	struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
	struct packet_offload *ptype;
	unsigned int grehlen = sizeof(*greh);
	int err = -ENOENT;
	__be16 type;

	skb->encapsulation = 1;
	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;

	type = greh->protocol;
	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	rcu_read_lock();
	ptype = gro_find_complete_by_type(type);
	if (ptype)
		err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);

	rcu_read_unlock();

	skb_set_inner_mac_header(skb, nhoff + grehlen);

	return err;
}

static const struct net_offload gre_offload = {
	.callbacks = {
		.gso_segment = gre_gso_segment,
		.gro_receive = gre_gro_receive,
		.gro_complete = gre_gro_complete,
	},
};

static int __init gre_offload_init(void)
{
	return inet_add_offload(&gre_offload, IPPROTO_GRE);
}
device_initcall(gre_offload_init);
Commit	Line	Data
c50cd357 DB	1	/*
	2	* IPV4 GSO/GRO offload support
	3	* Linux INET implementation
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public License
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the License, or (at your option) any later version.
	9	*
	10	* GRE GSO support
	11	*/
	12
	13	#include <linux/skbuff.h>
cf172283	14	#include <linux/init.h>
c50cd357 DB	15	#include <net/protocol.h>
	16	#include <net/gre.h>
	17
c50cd357 DB	18	static struct sk_buff gre_gso_segment(struct sk_buff skb,
	19	netdev_features_t features)
	20	{
2e598af7	21	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
c50cd357	22	struct sk_buff *segs = ERR_PTR(-EINVAL);
7a7ffbab	23	u16 mac_offset = skb->mac_header;
c50cd357	24	__be16 protocol = skb->protocol;
2e598af7 AD	25	u16 mac_len = skb->mac_len;
	26	int gre_offset, outer_hlen;
	27	bool need_csum;
c50cd357 DB	28
	29	if (unlikely(skb_shinfo(skb)->gso_type &
	30	~(SKB_GSO_TCPV4 \|
	31	SKB_GSO_TCPV6 \|
	32	SKB_GSO_UDP \|
	33	SKB_GSO_DODGY \|
	34	SKB_GSO_TCP_ECN \|
cb32f511	35	SKB_GSO_GRE \|
4749c09c	36	SKB_GSO_GRE_CSUM \|
7e3b6e74 ED	37	SKB_GSO_IPIP \|
7e3b6e74 ED	38	SKB_GSO_SIT)))
c50cd357 DB	39	goto out;
c50cd357 DB	40
53e50398 TH	41	if (!skb->encapsulation)
	42	goto out;
	43
2e598af7	44	if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr)))
c50cd357 DB	45	goto out;
c50cd357 DB	46
2e598af7	47	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
b884b1a4 NC	48	goto out;
b884b1a4 NC	49
c50cd357	50	/* setup inner skb. */
c50cd357	51	skb->encapsulation = 0;
2e598af7	52	__skb_pull(skb, tnl_hlen);
c50cd357 DB	53	skb_reset_mac_header(skb);
	54	skb_set_network_header(skb, skb_inner_network_offset(skb));
	55	skb->mac_len = skb_inner_network_offset(skb);
38720352	56	skb->protocol = skb->inner_protocol;
c50cd357	57
2e598af7 AD	58	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM);
	59	skb->encap_hdr_csum = need_csum;
	60
bef3c6c9 AD	61	features &= skb->dev->hw_enc_features;
bef3c6c9 AD	62
c50cd357	63	/* segment inner packet. */
bef3c6c9	64	segs = skb_mac_gso_segment(skb, features);
5a8dbf03	65	if (IS_ERR_OR_NULL(segs)) {
2e598af7 AD	66	skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
2e598af7 AD	67	mac_len);
c50cd357	68	goto out;
7a7ffbab	69	}
c50cd357	70
2e598af7 AD	71	outer_hlen = skb_tnl_header_len(skb);
2e598af7 AD	72	gre_offset = outer_hlen - tnl_hlen;
c50cd357	73	skb = segs;
c50cd357	74	do {
38720352	75	struct gre_base_hdr *greh;
2e598af7	76	__be32 *pcsum;
c50cd357	77
cdbaa0bb AD	78	skb_reset_inner_headers(skb);
	79	skb->encapsulation = 1;
	80
c50cd357 DB	81	skb->mac_len = mac_len;
c50cd357 DB	82	skb->protocol = protocol;
2e598af7 AD	83
	84	__skb_push(skb, outer_hlen);
	85	skb_reset_mac_header(skb);
	86	skb_set_network_header(skb, mac_len);
	87	skb_set_transport_header(skb, gre_offset);
	88
	89	if (!need_csum)
	90	continue;
	91
	92	greh = (struct gre_base_hdr *)skb_transport_header(skb);
	93	pcsum = (__be32 *)(greh + 1);
	94
	95	*pcsum = 0;
	96	(__sum16 )pcsum = gso_make_checksum(skb, 0);
c50cd357 DB	97	} while ((skb = skb->next));
	98	out:
	99	return segs;
	100	}
	101
bf5a755f JC	102	static struct sk_buff gre_gro_receive(struct sk_buff head,
	103	struct sk_buff *skb)
	104	{
	105	struct sk_buff **pp = NULL;
	106	struct sk_buff *p;
	107	const struct gre_base_hdr *greh;
	108	unsigned int hlen, grehlen;
	109	unsigned int off;
	110	int flush = 1;
	111	struct packet_offload *ptype;
	112	__be16 type;
	113
	114	off = skb_gro_offset(skb);
	115	hlen = off + sizeof(*greh);
	116	greh = skb_gro_header_fast(skb, off);
	117	if (skb_gro_header_hard(skb, hlen)) {
	118	greh = skb_gro_header_slow(skb, hlen, off);
	119	if (unlikely(!greh))
	120	goto out;
	121	}
	122
	123	/* Only support version 0 and K (key), C (csum) flags. Note that
	124	* although the support for the S (seq#) flag can be added easily
	125	* for GRO, this is problematic for GSO hence can not be enabled
	126	* here because a GRO pkt may end up in the forwarding path, thus
	127	* requiring GSO support to break it up correctly.
	128	*/
	129	if ((greh->flags & ~(GRE_KEY\|GRE_CSUM)) != 0)
	130	goto out;
	131
	132	type = greh->protocol;
	133
	134	rcu_read_lock();
	135	ptype = gro_find_receive_by_type(type);
51456b29	136	if (!ptype)
bf5a755f JC	137	goto out_unlock;
	138
	139	grehlen = GRE_HEADER_SECTION;
	140
	141	if (greh->flags & GRE_KEY)
	142	grehlen += GRE_HEADER_SECTION;
	143
	144	if (greh->flags & GRE_CSUM)
	145	grehlen += GRE_HEADER_SECTION;
	146
	147	hlen = off + grehlen;
	148	if (skb_gro_header_hard(skb, hlen)) {
	149	greh = skb_gro_header_slow(skb, hlen, off);
	150	if (unlikely(!greh))
	151	goto out_unlock;
	152	}
758f75d1 TH	153
758f75d1 TH	154	/* Don't bother verifying checksum if we're going to flush anyway. */
884d338c TH	155	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
884d338c TH	156	if (skb_gro_checksum_simple_validate(skb))
bf5a755f	157	goto out_unlock;
758f75d1	158
884d338c TH	159	skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0,
	160	null_compute_pseudo);
	161	}
	162
bf5a755f JC	163	flush = 0;
	164
	165	for (p = *head; p; p = p->next) {
	166	const struct gre_base_hdr *greh2;
	167
	168	if (!NAPI_GRO_CB(p)->same_flow)
	169	continue;
	170
	171	/* The following checks are needed to ensure only pkts
	172	* from the same tunnel are considered for aggregation.
	173	* The criteria for "the same tunnel" includes:
	174	* 1) same version (we only support version 0 here)
	175	* 2) same protocol (we only support ETH_P_IP for now)
	176	* 3) same set of flags
	177	* 4) same key if the key field is present.
	178	*/
	179	greh2 = (struct gre_base_hdr *)(p->data + off);
	180
	181	if (greh2->flags != greh->flags \|\|
	182	greh2->protocol != greh->protocol) {
	183	NAPI_GRO_CB(p)->same_flow = 0;
	184	continue;
	185	}
	186	if (greh->flags & GRE_KEY) {
	187	/* compare keys */
	188	if ((__be32 )(greh2+1) != (__be32 )(greh+1)) {
	189	NAPI_GRO_CB(p)->same_flow = 0;
	190	continue;
	191	}
	192	}
	193	}
	194
	195	skb_gro_pull(skb, grehlen);
	196
	197	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	198	skb_gro_postpull_rcsum(skb, greh, grehlen);
	199
	200	pp = ptype->callbacks.gro_receive(head, skb);
	201
	202	out_unlock:
	203	rcu_read_unlock();
	204	out:
	205	NAPI_GRO_CB(skb)->flush \|= flush;
	206
	207	return pp;
	208	}
	209
d10dbad2	210	static int gre_gro_complete(struct sk_buff *skb, int nhoff)
bf5a755f JC	211	{
	212	struct gre_base_hdr greh = (struct gre_base_hdr )(skb->data + nhoff);
	213	struct packet_offload *ptype;
	214	unsigned int grehlen = sizeof(*greh);
	215	int err = -ENOENT;
	216	__be16 type;
	217
c3caf119 JC	218	skb->encapsulation = 1;
	219	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
	220
bf5a755f JC	221	type = greh->protocol;
	222	if (greh->flags & GRE_KEY)
	223	grehlen += GRE_HEADER_SECTION;
	224
	225	if (greh->flags & GRE_CSUM)
	226	grehlen += GRE_HEADER_SECTION;
	227
	228	rcu_read_lock();
	229	ptype = gro_find_complete_by_type(type);
00db4124	230	if (ptype)
bf5a755f JC	231	err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
	232
	233	rcu_read_unlock();
6fb2a756 TH	234
	235	skb_set_inner_mac_header(skb, nhoff + grehlen);
	236
bf5a755f JC	237	return err;
	238	}
	239
c50cd357 DB	240	static const struct net_offload gre_offload = {
c50cd357 DB	241	.callbacks = {
c50cd357	242	.gso_segment = gre_gso_segment,
bf5a755f JC	243	.gro_receive = gre_gro_receive,
bf5a755f JC	244	.gro_complete = gre_gro_complete,
c50cd357 DB	245	},
	246	};
	247
438e38fa	248	static int __init gre_offload_init(void)
c50cd357 DB	249	{
	250	return inet_add_offload(&gre_offload, IPPROTO_GRE);
	251	}
cf172283	252	device_initcall(gre_offload_init);