[deliverable/linux.git] / drivers / net / sfc / filter.c

/****************************************************************************
 * Driver for Solarflare Solarstorm network controllers and boards
 * Copyright 2005-2010 Solarflare Communications Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation, incorporated herein by reference.
 */

#include <linux/in.h>
#include "efx.h"
#include "filter.h"
#include "io.h"
#include "nic.h"
#include "regs.h"

/* "Fudge factors" - difference between programmed value and actual depth.
 * Due to pipelined implementation we need to program H/W with a value that
 * is larger than the hop limit we want.
 */
#define FILTER_CTL_SRCH_FUDGE_WILD 3
#define FILTER_CTL_SRCH_FUDGE_FULL 1

/* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 * We also need to avoid infinite loops in efx_filter_search() when the
 * table is full.
 */
#define FILTER_CTL_SRCH_MAX 200

enum efx_filter_table_id {
	EFX_FILTER_TABLE_RX_IP = 0,
	EFX_FILTER_TABLE_RX_MAC,
	EFX_FILTER_TABLE_COUNT,
};

struct efx_filter_table {
	enum efx_filter_table_id id;
	u32		offset;		/* address of table relative to BAR */
	unsigned	size;		/* number of entries */
	unsigned	step;		/* step between entries */
	unsigned	used;		/* number currently used */
	unsigned long	*used_bitmap;
	struct efx_filter_spec *spec;
	unsigned	search_depth[EFX_FILTER_TYPE_COUNT];
};

struct efx_filter_state {
	spinlock_t	lock;
	struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
};

/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
static u16 efx_filter_hash(u32 key)
{
	u16 tmp;

	/* First 16 rounds */
	tmp = 0x1fff ^ key >> 16;
	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	tmp = tmp ^ tmp >> 9;
	/* Last 16 rounds */
	tmp = tmp ^ tmp << 13 ^ key;
	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	return tmp ^ tmp >> 9;
}

/* To allow for hash collisions, filter search continues at these
 * increments from the first possible entry selected by the hash. */
static u16 efx_filter_increment(u32 key)
{
	return key * 2 - 1;
}

static enum efx_filter_table_id
efx_filter_spec_table_id(const struct efx_filter_spec *spec)
{
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
	EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
	return spec->type >> 2;
}

static struct efx_filter_table *
efx_filter_spec_table(struct efx_filter_state *state,
		      const struct efx_filter_spec *spec)
{
	if (spec->type == EFX_FILTER_UNSPEC)
		return NULL;
	else
		return &state->table[efx_filter_spec_table_id(spec)];
}

static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
{
	memset(table->search_depth, 0, sizeof(table->search_depth));
}

static void efx_filter_push_rx_limits(struct efx_nic *efx)
{
	struct efx_filter_state *state = efx->filter_state;
	struct efx_filter_table *table;
	efx_oword_t filter_ctl;

	efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);

	table = &state->table[EFX_FILTER_TABLE_RX_IP];
	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
			    table->search_depth[EFX_FILTER_TCP_FULL] +
			    FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
			    table->search_depth[EFX_FILTER_TCP_WILD] +
			    FILTER_CTL_SRCH_FUDGE_WILD);
	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
			    table->search_depth[EFX_FILTER_UDP_FULL] +
			    FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
			    table->search_depth[EFX_FILTER_UDP_WILD] +
			    FILTER_CTL_SRCH_FUDGE_WILD);

	table = &state->table[EFX_FILTER_TABLE_RX_MAC];
	if (table->size) {
		EFX_SET_OWORD_FIELD(
			filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
			table->search_depth[EFX_FILTER_MAC_FULL] +
			FILTER_CTL_SRCH_FUDGE_FULL);
		EFX_SET_OWORD_FIELD(
			filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
			table->search_depth[EFX_FILTER_MAC_WILD] +
			FILTER_CTL_SRCH_FUDGE_WILD);
	}

	efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
}

static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
					 __be32 host1, __be16 port1,
					 __be32 host2, __be16 port2)
{
	spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
	spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
	spec->data[2] = ntohl(host2);
}

/**
 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 * @spec: Specification to initialise
 * @proto: Transport layer protocol number
 * @host: Local host address (network byte order)
 * @port: Local port (network byte order)
 */
int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
			      __be32 host, __be16 port)
{
	__be32 host1;
	__be16 port1;

	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));

	/* This cannot currently be combined with other filtering */
	if (spec->type != EFX_FILTER_UNSPEC)
		return -EPROTONOSUPPORT;

	if (port == 0)
		return -EINVAL;

	switch (proto) {
	case IPPROTO_TCP:
		spec->type = EFX_FILTER_TCP_WILD;
		break;
	case IPPROTO_UDP:
		spec->type = EFX_FILTER_UDP_WILD;
		break;
	default:
		return -EPROTONOSUPPORT;
	}

	/* Filter is constructed in terms of source and destination,
	 * with the odd wrinkle that the ports are swapped in a UDP
	 * wildcard filter.  We need to convert from local and remote
	 * (= zero for wildcard) addresses.
	 */
	host1 = 0;
	if (proto != IPPROTO_UDP) {
		port1 = 0;
	} else {
		port1 = port;
		port = 0;
	}

	__efx_filter_set_ipv4(spec, host1, port1, host, port);
	return 0;
}

/**
 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 * @spec: Specification to initialise
 * @proto: Transport layer protocol number
 * @host: Local host address (network byte order)
 * @port: Local port (network byte order)
 * @rhost: Remote host address (network byte order)
 * @rport: Remote port (network byte order)
 */
int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
			     __be32 host, __be16 port,
			     __be32 rhost, __be16 rport)
{
	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));

	/* This cannot currently be combined with other filtering */
	if (spec->type != EFX_FILTER_UNSPEC)
		return -EPROTONOSUPPORT;

	if (port == 0 || rport == 0)
		return -EINVAL;

	switch (proto) {
	case IPPROTO_TCP:
		spec->type = EFX_FILTER_TCP_FULL;
		break;
	case IPPROTO_UDP:
		spec->type = EFX_FILTER_UDP_FULL;
		break;
	default:
		return -EPROTONOSUPPORT;
	}

	__efx_filter_set_ipv4(spec, rhost, rport, host, port);
	return 0;
}

/**
 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 * @spec: Specification to initialise
 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 * @addr: Local Ethernet MAC address
 */
int efx_filter_set_eth_local(struct efx_filter_spec *spec,
			     u16 vid, const u8 *addr)
{
	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));

	/* This cannot currently be combined with other filtering */
	if (spec->type != EFX_FILTER_UNSPEC)
		return -EPROTONOSUPPORT;

	if (vid == EFX_FILTER_VID_UNSPEC) {
		spec->type = EFX_FILTER_MAC_WILD;
		spec->data[0] = 0;
	} else {
		spec->type = EFX_FILTER_MAC_FULL;
		spec->data[0] = vid;
	}

	spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
	spec->data[2] = addr[0] << 8 | addr[1];
	return 0;
}

/* Build a filter entry and return its n-tuple key. */
static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
{
	u32 data3;

	switch (efx_filter_spec_table_id(spec)) {
	case EFX_FILTER_TABLE_RX_IP: {
		bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
			       spec->type == EFX_FILTER_UDP_WILD);
		EFX_POPULATE_OWORD_7(
			*filter,
			FRF_BZ_RSS_EN,
			!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
			FRF_BZ_SCATTER_EN,
			!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
			FRF_BZ_TCP_UDP, is_udp,
			FRF_BZ_RXQ_ID, spec->dmaq_id,
			EFX_DWORD_2, spec->data[2],
			EFX_DWORD_1, spec->data[1],
			EFX_DWORD_0, spec->data[0]);
		data3 = is_udp;
		break;
	}

	case EFX_FILTER_TABLE_RX_MAC: {
		bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
		EFX_POPULATE_OWORD_8(
			*filter,
			FRF_CZ_RMFT_RSS_EN,
			!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
			FRF_CZ_RMFT_SCATTER_EN,
			!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
			FRF_CZ_RMFT_IP_OVERRIDE,
			!!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
			FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
			FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
			FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
			FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
			FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
		data3 = is_wild;
		break;
	}

	default:
		BUG();
	}

	return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
}

static bool efx_filter_equal(const struct efx_filter_spec *left,
			     const struct efx_filter_spec *right)
{
	if (left->type != right->type ||
	    memcmp(left->data, right->data, sizeof(left->data)))
		return false;

	return true;
}

static int efx_filter_search(struct efx_filter_table *table,
			     struct efx_filter_spec *spec, u32 key,
			     bool for_insert, int *depth_required)
{
	unsigned hash, incr, filter_idx, depth;
	struct efx_filter_spec *cmp;

	hash = efx_filter_hash(key);
	incr = efx_filter_increment(key);

	for (depth = 1, filter_idx = hash & (table->size - 1);
	     depth <= FILTER_CTL_SRCH_MAX &&
		     test_bit(filter_idx, table->used_bitmap);
	     ++depth) {
		cmp = &table->spec[filter_idx];
		if (efx_filter_equal(spec, cmp))
			goto found;
		filter_idx = (filter_idx + incr) & (table->size - 1);
	}
	if (!for_insert)
		return -ENOENT;
	if (depth > FILTER_CTL_SRCH_MAX)
		return -EBUSY;
found:
	*depth_required = depth;
	return filter_idx;
}

/* Construct/deconstruct external filter IDs */

static inline int
efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
{
	return table_id << 16 | index;
}

/**
 * efx_filter_insert_filter - add or replace a filter
 * @efx: NIC in which to insert the filter
 * @spec: Specification for the filter
 * @replace: Flag for whether the specified filter may replace a filter
 *	with an identical match expression and equal or lower priority
 *
 * On success, return the filter ID.
 * On failure, return a negative error code.
 */
int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
			     bool replace)
{
	struct efx_filter_state *state = efx->filter_state;
	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
	struct efx_filter_spec *saved_spec;
	efx_oword_t filter;
	int filter_idx, depth;
	u32 key;
	int rc;

	if (!table || table->size == 0)
		return -EINVAL;

	key = efx_filter_build(&filter, spec);

	netif_vdbg(efx, hw, efx->net_dev,
		   "%s: type %d search_depth=%d", __func__, spec->type,
		   table->search_depth[spec->type]);

	spin_lock_bh(&state->lock);

	rc = efx_filter_search(table, spec, key, true, &depth);
	if (rc < 0)
		goto out;
	filter_idx = rc;
	BUG_ON(filter_idx >= table->size);
	saved_spec = &table->spec[filter_idx];

	if (test_bit(filter_idx, table->used_bitmap)) {
		/* Should we replace the existing filter? */
		if (!replace) {
			rc = -EEXIST;
			goto out;
		}
		if (spec->priority < saved_spec->priority) {
			rc = -EPERM;
			goto out;
		}
	} else {
		__set_bit(filter_idx, table->used_bitmap);
		++table->used;
	}
	*saved_spec = *spec;

	if (table->search_depth[spec->type] < depth) {
		table->search_depth[spec->type] = depth;
		efx_filter_push_rx_limits(efx);
	}

	efx_writeo(efx, &filter, table->offset + table->step * filter_idx);

	netif_vdbg(efx, hw, efx->net_dev,
		   "%s: filter type %d index %d rxq %u set",
		   __func__, spec->type, filter_idx, spec->dmaq_id);
	rc = efx_filter_make_id(table->id, filter_idx);

out:
	spin_unlock_bh(&state->lock);
	return rc;
}

static void efx_filter_table_clear_entry(struct efx_nic *efx,
					 struct efx_filter_table *table,
					 int filter_idx)
{
	static efx_oword_t filter;

	if (test_bit(filter_idx, table->used_bitmap)) {
		__clear_bit(filter_idx, table->used_bitmap);
		--table->used;
		memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));

		efx_writeo(efx, &filter,
			   table->offset + table->step * filter_idx);
	}
}

/**
 * efx_filter_remove_filter - remove a filter by specification
 * @efx: NIC from which to remove the filter
 * @spec: Specification for the filter
 *
 * On success, return zero.
 * On failure, return a negative error code.
 */
int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
{
	struct efx_filter_state *state = efx->filter_state;
	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
	struct efx_filter_spec *saved_spec;
	efx_oword_t filter;
	int filter_idx, depth;
	u32 key;
	int rc;

	if (!table)
		return -EINVAL;

	key = efx_filter_build(&filter, spec);

	spin_lock_bh(&state->lock);

	rc = efx_filter_search(table, spec, key, false, &depth);
	if (rc < 0)
		goto out;
	filter_idx = rc;
	saved_spec = &table->spec[filter_idx];

	if (spec->priority < saved_spec->priority) {
		rc = -EPERM;
		goto out;
	}

	efx_filter_table_clear_entry(efx, table, filter_idx);
	if (table->used == 0)
		efx_filter_table_reset_search_depth(table);
	rc = 0;

out:
	spin_unlock_bh(&state->lock);
	return rc;
}

static void efx_filter_table_clear(struct efx_nic *efx,
				   enum efx_filter_table_id table_id,
				   enum efx_filter_priority priority)
{
	struct efx_filter_state *state = efx->filter_state;
	struct efx_filter_table *table = &state->table[table_id];
	int filter_idx;

	spin_lock_bh(&state->lock);

	for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
		if (table->spec[filter_idx].priority <= priority)
			efx_filter_table_clear_entry(efx, table, filter_idx);
	if (table->used == 0)
		efx_filter_table_reset_search_depth(table);

	spin_unlock_bh(&state->lock);
}

/**
 * efx_filter_clear_rx - remove RX filters by priority
 * @efx: NIC from which to remove the filters
 * @priority: Maximum priority to remove
 */
void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
{
	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
}

/* Restore filter stater after reset */
void efx_restore_filters(struct efx_nic *efx)
{
	struct efx_filter_state *state = efx->filter_state;
	enum efx_filter_table_id table_id;
	struct efx_filter_table *table;
	efx_oword_t filter;
	int filter_idx;

	spin_lock_bh(&state->lock);

	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
		table = &state->table[table_id];
		for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
			if (!test_bit(filter_idx, table->used_bitmap))
				continue;
			efx_filter_build(&filter, &table->spec[filter_idx]);
			efx_writeo(efx, &filter,
				   table->offset + table->step * filter_idx);
		}
	}

	efx_filter_push_rx_limits(efx);

	spin_unlock_bh(&state->lock);
}

int efx_probe_filters(struct efx_nic *efx)
{
	struct efx_filter_state *state;
	struct efx_filter_table *table;
	unsigned table_id;

	state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
	if (!state)
		return -ENOMEM;
	efx->filter_state = state;

	spin_lock_init(&state->lock);

	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
		table = &state->table[EFX_FILTER_TABLE_RX_IP];
		table->id = EFX_FILTER_TABLE_RX_IP;
		table->offset = FR_BZ_RX_FILTER_TBL0;
		table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
		table->step = FR_BZ_RX_FILTER_TBL0_STEP;
	}

	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
		table = &state->table[EFX_FILTER_TABLE_RX_MAC];
		table->id = EFX_FILTER_TABLE_RX_MAC;
		table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
		table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
		table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
	}

	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
		table = &state->table[table_id];
		if (table->size == 0)
			continue;
		table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
					     sizeof(unsigned long),
					     GFP_KERNEL);
		if (!table->used_bitmap)
			goto fail;
		table->spec = vzalloc(table->size * sizeof(*table->spec));
		if (!table->spec)
			goto fail;
	}

	return 0;

fail:
	efx_remove_filters(efx);
	return -ENOMEM;
}

void efx_remove_filters(struct efx_nic *efx)
{
	struct efx_filter_state *state = efx->filter_state;
	enum efx_filter_table_id table_id;

	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
		kfree(state->table[table_id].used_bitmap);
		vfree(state->table[table_id].spec);
	}
	kfree(state);
}
Commit	Line	Data
64eebcfd BH	1	/****************************************************************************
	2	* Driver for Solarflare Solarstorm network controllers and boards
	3	* Copyright 2005-2010 Solarflare Communications Inc.
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published
	7	* by the Free Software Foundation, incorporated herein by reference.
	8	*/
	9
c39d35eb	10	#include <linux/in.h>
64eebcfd BH	11	#include "efx.h"
	12	#include "filter.h"
	13	#include "io.h"
	14	#include "nic.h"
	15	#include "regs.h"
	16
	17	/* "Fudge factors" - difference between programmed value and actual depth.
	18	* Due to pipelined implementation we need to program H/W with a value that
	19	* is larger than the hop limit we want.
	20	*/
	21	#define FILTER_CTL_SRCH_FUDGE_WILD 3
	22	#define FILTER_CTL_SRCH_FUDGE_FULL 1
	23
993284df BH	24	/* Hard maximum hop limit. Hardware will time-out beyond 200-something.
	25	* We also need to avoid infinite loops in efx_filter_search() when the
	26	* table is full.
	27	*/
	28	#define FILTER_CTL_SRCH_MAX 200
	29
8891681a BH	30	enum efx_filter_table_id {
	31	EFX_FILTER_TABLE_RX_IP = 0,
	32	EFX_FILTER_TABLE_RX_MAC,
	33	EFX_FILTER_TABLE_COUNT,
	34	};
	35
64eebcfd	36	struct efx_filter_table {
c39d35eb	37	enum efx_filter_table_id id;
64eebcfd BH	38	u32 offset; /* address of table relative to BAR */
	39	unsigned size; /* number of entries */
	40	unsigned step; /* step between entries */
	41	unsigned used; /* number currently used */
	42	unsigned long *used_bitmap;
	43	struct efx_filter_spec *spec;
c39d35eb	44	unsigned search_depth[EFX_FILTER_TYPE_COUNT];
64eebcfd BH	45	};
	46
	47	struct efx_filter_state {
	48	spinlock_t lock;
	49	struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
64eebcfd BH	50	};
	51
	52	/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
	53	* key derived from the n-tuple. The initial LFSR state is 0xffff. */
	54	static u16 efx_filter_hash(u32 key)
	55	{
	56	u16 tmp;
	57
	58	/* First 16 rounds */
	59	tmp = 0x1fff ^ key >> 16;
	60	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	61	tmp = tmp ^ tmp >> 9;
	62	/* Last 16 rounds */
	63	tmp = tmp ^ tmp << 13 ^ key;
	64	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	65	return tmp ^ tmp >> 9;
	66	}
	67
	68	/* To allow for hash collisions, filter search continues at these
	69	* increments from the first possible entry selected by the hash. */
	70	static u16 efx_filter_increment(u32 key)
	71	{
	72	return key * 2 - 1;
	73	}
	74
	75	static enum efx_filter_table_id
c39d35eb BH	76	efx_filter_spec_table_id(const struct efx_filter_spec *spec)
	77	{
	78	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
	79	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
	80	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
	81	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
	82	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
	83	BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
	84	EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
	85	return spec->type >> 2;
	86	}
	87
	88	static struct efx_filter_table *
	89	efx_filter_spec_table(struct efx_filter_state *state,
	90	const struct efx_filter_spec *spec)
64eebcfd	91	{
c39d35eb BH	92	if (spec->type == EFX_FILTER_UNSPEC)
	93	return NULL;
	94	else
	95	return &state->table[efx_filter_spec_table_id(spec)];
64eebcfd BH	96	}
64eebcfd BH	97
c39d35eb	98	static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
64eebcfd	99	{
c39d35eb	100	memset(table->search_depth, 0, sizeof(table->search_depth));
64eebcfd BH	101	}
	102
	103	static void efx_filter_push_rx_limits(struct efx_nic *efx)
	104	{
	105	struct efx_filter_state *state = efx->filter_state;
c39d35eb	106	struct efx_filter_table *table;
64eebcfd BH	107	efx_oword_t filter_ctl;
	108
	109	efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
	110
c39d35eb	111	table = &state->table[EFX_FILTER_TABLE_RX_IP];
64eebcfd	112	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
c39d35eb	113	table->search_depth[EFX_FILTER_TCP_FULL] +
64eebcfd BH	114	FILTER_CTL_SRCH_FUDGE_FULL);
64eebcfd BH	115	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
c39d35eb	116	table->search_depth[EFX_FILTER_TCP_WILD] +
64eebcfd BH	117	FILTER_CTL_SRCH_FUDGE_WILD);
64eebcfd BH	118	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
c39d35eb	119	table->search_depth[EFX_FILTER_UDP_FULL] +
64eebcfd BH	120	FILTER_CTL_SRCH_FUDGE_FULL);
64eebcfd BH	121	EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
c39d35eb	122	table->search_depth[EFX_FILTER_UDP_WILD] +
64eebcfd BH	123	FILTER_CTL_SRCH_FUDGE_WILD);
64eebcfd BH	124
c39d35eb BH	125	table = &state->table[EFX_FILTER_TABLE_RX_MAC];
c39d35eb BH	126	if (table->size) {
64eebcfd BH	127	EFX_SET_OWORD_FIELD(
64eebcfd BH	128	filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
c39d35eb	129	table->search_depth[EFX_FILTER_MAC_FULL] +
64eebcfd BH	130	FILTER_CTL_SRCH_FUDGE_FULL);
	131	EFX_SET_OWORD_FIELD(
	132	filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
c39d35eb	133	table->search_depth[EFX_FILTER_MAC_WILD] +
64eebcfd BH	134	FILTER_CTL_SRCH_FUDGE_WILD);
	135	}
	136
	137	efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
	138	}
	139
c39d35eb BH	140	static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
	141	__be32 host1, __be16 port1,
	142	__be32 host2, __be16 port2)
	143	{
	144	spec->data[0] = ntohl(host1) << 16 \| ntohs(port1);
	145	spec->data[1] = ntohs(port2) << 16 \| ntohl(host1) >> 16;
	146	spec->data[2] = ntohl(host2);
	147	}
	148
	149	/**
	150	* efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
	151	* @spec: Specification to initialise
	152	* @proto: Transport layer protocol number
	153	* @host: Local host address (network byte order)
	154	* @port: Local port (network byte order)
	155	*/
	156	int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
	157	__be32 host, __be16 port)
	158	{
	159	__be32 host1;
	160	__be16 port1;
	161
	162	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
	163
	164	/* This cannot currently be combined with other filtering */
	165	if (spec->type != EFX_FILTER_UNSPEC)
	166	return -EPROTONOSUPPORT;
	167
	168	if (port == 0)
	169	return -EINVAL;
	170
	171	switch (proto) {
	172	case IPPROTO_TCP:
	173	spec->type = EFX_FILTER_TCP_WILD;
	174	break;
	175	case IPPROTO_UDP:
	176	spec->type = EFX_FILTER_UDP_WILD;
	177	break;
	178	default:
	179	return -EPROTONOSUPPORT;
	180	}
	181
	182	/* Filter is constructed in terms of source and destination,
	183	* with the odd wrinkle that the ports are swapped in a UDP
	184	* wildcard filter. We need to convert from local and remote
	185	* (= zero for wildcard) addresses.
	186	*/
	187	host1 = 0;
	188	if (proto != IPPROTO_UDP) {
	189	port1 = 0;
	190	} else {
	191	port1 = port;
	192	port = 0;
	193	}
	194
	195	__efx_filter_set_ipv4(spec, host1, port1, host, port);
	196	return 0;
	197	}
	198
	199	/**
	200	* efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
	201	* @spec: Specification to initialise
	202	* @proto: Transport layer protocol number
	203	* @host: Local host address (network byte order)
204	* @port: Local port (network byte order)
205	* @rhost: Remote host address (network byte order)
206	* @rport: Remote port (network byte order)
207	*/
208	int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
209	__be32 host, __be16 port,
210	__be32 rhost, __be16 rport)
211	{
212	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
213
214	/* This cannot currently be combined with other filtering */
215	if (spec->type != EFX_FILTER_UNSPEC)
216	return -EPROTONOSUPPORT;
217
218	if (port == 0 \|\| rport == 0)
219	return -EINVAL;
220
221	switch (proto) {
222	case IPPROTO_TCP:
223	spec->type = EFX_FILTER_TCP_FULL;
224	break;
225	case IPPROTO_UDP:
226	spec->type = EFX_FILTER_UDP_FULL;
227	break;
228	default:
229	return -EPROTONOSUPPORT;
230	}
231
232	__efx_filter_set_ipv4(spec, rhost, rport, host, port);
233	return 0;
234	}
235
236	/**
237	* efx_filter_set_eth_local - specify local Ethernet address and optional VID
238	* @spec: Specification to initialise
239	* @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
240	* @addr: Local Ethernet MAC address
241	*/
242	int efx_filter_set_eth_local(struct efx_filter_spec *spec,
243	u16 vid, const u8 *addr)
244	{
245	EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
246
247	/* This cannot currently be combined with other filtering */
248	if (spec->type != EFX_FILTER_UNSPEC)
249	return -EPROTONOSUPPORT;
250
251	if (vid == EFX_FILTER_VID_UNSPEC) {
252	spec->type = EFX_FILTER_MAC_WILD;
253	spec->data[0] = 0;
254	} else {
255	spec->type = EFX_FILTER_MAC_FULL;
256	spec->data[0] = vid;
257	}
258
259	spec->data[1] = addr[2] << 24 \| addr[3] << 16 \| addr[4] << 8 \| addr[5];
260	spec->data[2] = addr[0] << 8 \| addr[1];
261	return 0;
262	}
263
64eebcfd BH	264	/* Build a filter entry and return its n-tuple key. */
	265	static u32 efx_filter_build(efx_oword_t filter, struct efx_filter_spec spec)
	266	{
	267	u32 data3;
	268
c39d35eb	269	switch (efx_filter_spec_table_id(spec)) {
64eebcfd	270	case EFX_FILTER_TABLE_RX_IP: {
c39d35eb BH	271	bool is_udp = (spec->type == EFX_FILTER_UDP_FULL \|\|
c39d35eb BH	272	spec->type == EFX_FILTER_UDP_WILD);
64eebcfd BH	273	EFX_POPULATE_OWORD_7(
	274	*filter,
	275	FRF_BZ_RSS_EN,
	276	!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
	277	FRF_BZ_SCATTER_EN,
	278	!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
	279	FRF_BZ_TCP_UDP, is_udp,
	280	FRF_BZ_RXQ_ID, spec->dmaq_id,
	281	EFX_DWORD_2, spec->data[2],
	282	EFX_DWORD_1, spec->data[1],
	283	EFX_DWORD_0, spec->data[0]);
	284	data3 = is_udp;
	285	break;
	286	}
	287
	288	case EFX_FILTER_TABLE_RX_MAC: {
c39d35eb	289	bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
64eebcfd BH	290	EFX_POPULATE_OWORD_8(
	291	*filter,
	292	FRF_CZ_RMFT_RSS_EN,
	293	!!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
	294	FRF_CZ_RMFT_SCATTER_EN,
	295	!!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
	296	FRF_CZ_RMFT_IP_OVERRIDE,
	297	!!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
	298	FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
	299	FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
	300	FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
	301	FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
	302	FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
	303	data3 = is_wild;
	304	break;
	305	}
	306
	307	default:
	308	BUG();
	309	}
	310
	311	return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
	312	}
	313
	314	static bool efx_filter_equal(const struct efx_filter_spec *left,
	315	const struct efx_filter_spec *right)
	316	{
	317	if (left->type != right->type \|\|
	318	memcmp(left->data, right->data, sizeof(left->data)))
	319	return false;
	320
	321	return true;
	322	}
	323
	324	static int efx_filter_search(struct efx_filter_table *table,
	325	struct efx_filter_spec *spec, u32 key,
	326	bool for_insert, int *depth_required)
	327	{
	328	unsigned hash, incr, filter_idx, depth;
	329	struct efx_filter_spec *cmp;
	330
	331	hash = efx_filter_hash(key);
	332	incr = efx_filter_increment(key);
	333
	334	for (depth = 1, filter_idx = hash & (table->size - 1);
993284df BH	335	depth <= FILTER_CTL_SRCH_MAX &&
993284df BH	336	test_bit(filter_idx, table->used_bitmap);
64eebcfd BH	337	++depth) {
	338	cmp = &table->spec[filter_idx];
	339	if (efx_filter_equal(spec, cmp))
	340	goto found;
	341	filter_idx = (filter_idx + incr) & (table->size - 1);
	342	}
	343	if (!for_insert)
	344	return -ENOENT;
993284df BH	345	if (depth > FILTER_CTL_SRCH_MAX)
993284df BH	346	return -EBUSY;
64eebcfd BH	347	found:
	348	*depth_required = depth;
	349	return filter_idx;
	350	}
	351
8891681a BH	352	/* Construct/deconstruct external filter IDs */
	353
	354	static inline int
	355	efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
	356	{
	357	return table_id << 16 \| index;
	358	}
	359
64eebcfd BH	360	/**
	361	* efx_filter_insert_filter - add or replace a filter
	362	* @efx: NIC in which to insert the filter
	363	* @spec: Specification for the filter
	364	* @replace: Flag for whether the specified filter may replace a filter
	365	* with an identical match expression and equal or lower priority
	366	*
8891681a	367	* On success, return the filter ID.
64eebcfd BH	368	* On failure, return a negative error code.
	369	*/
	370	int efx_filter_insert_filter(struct efx_nic efx, struct efx_filter_spec spec,
	371	bool replace)
	372	{
	373	struct efx_filter_state *state = efx->filter_state;
c39d35eb	374	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
64eebcfd BH	375	struct efx_filter_spec *saved_spec;
	376	efx_oword_t filter;
	377	int filter_idx, depth;
	378	u32 key;
	379	int rc;
	380
c39d35eb	381	if (!table \|\| table->size == 0)
64eebcfd BH	382	return -EINVAL;
	383
	384	key = efx_filter_build(&filter, spec);
	385
	386	netif_vdbg(efx, hw, efx->net_dev,
	387	"%s: type %d search_depth=%d", __func__, spec->type,
c39d35eb	388	table->search_depth[spec->type]);
64eebcfd BH	389
	390	spin_lock_bh(&state->lock);
	391
	392	rc = efx_filter_search(table, spec, key, true, &depth);
	393	if (rc < 0)
	394	goto out;
	395	filter_idx = rc;
	396	BUG_ON(filter_idx >= table->size);
	397	saved_spec = &table->spec[filter_idx];
	398
	399	if (test_bit(filter_idx, table->used_bitmap)) {
	400	/* Should we replace the existing filter? */
	401	if (!replace) {
	402	rc = -EEXIST;
	403	goto out;
	404	}
	405	if (spec->priority < saved_spec->priority) {
	406	rc = -EPERM;
	407	goto out;
	408	}
	409	} else {
	410	__set_bit(filter_idx, table->used_bitmap);
	411	++table->used;
	412	}
	413	saved_spec = spec;
	414
c39d35eb BH	415	if (table->search_depth[spec->type] < depth) {
c39d35eb BH	416	table->search_depth[spec->type] = depth;
64eebcfd BH	417	efx_filter_push_rx_limits(efx);
	418	}
	419
	420	efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
	421
	422	netif_vdbg(efx, hw, efx->net_dev,
	423	"%s: filter type %d index %d rxq %u set",
	424	__func__, spec->type, filter_idx, spec->dmaq_id);
c39d35eb	425	rc = efx_filter_make_id(table->id, filter_idx);
64eebcfd BH	426
	427	out:
	428	spin_unlock_bh(&state->lock);
	429	return rc;
	430	}
	431
	432	static void efx_filter_table_clear_entry(struct efx_nic *efx,
	433	struct efx_filter_table *table,
	434	int filter_idx)
	435	{
	436	static efx_oword_t filter;
	437
	438	if (test_bit(filter_idx, table->used_bitmap)) {
	439	__clear_bit(filter_idx, table->used_bitmap);
	440	--table->used;
	441	memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
	442
	443	efx_writeo(efx, &filter,
	444	table->offset + table->step * filter_idx);
	445	}
	446	}
	447
	448	/**
	449	* efx_filter_remove_filter - remove a filter by specification
	450	* @efx: NIC from which to remove the filter
	451	* @spec: Specification for the filter
	452	*
	453	* On success, return zero.
	454	* On failure, return a negative error code.
	455	*/
	456	int efx_filter_remove_filter(struct efx_nic efx, struct efx_filter_spec spec)
	457	{
	458	struct efx_filter_state *state = efx->filter_state;
c39d35eb	459	struct efx_filter_table *table = efx_filter_spec_table(state, spec);
64eebcfd BH	460	struct efx_filter_spec *saved_spec;
	461	efx_oword_t filter;
	462	int filter_idx, depth;
	463	u32 key;
	464	int rc;
	465
c39d35eb BH	466	if (!table)
	467	return -EINVAL;
	468
64eebcfd BH	469	key = efx_filter_build(&filter, spec);
	470
	471	spin_lock_bh(&state->lock);
	472
	473	rc = efx_filter_search(table, spec, key, false, &depth);
	474	if (rc < 0)
	475	goto out;
	476	filter_idx = rc;
	477	saved_spec = &table->spec[filter_idx];
	478
	479	if (spec->priority < saved_spec->priority) {
	480	rc = -EPERM;
	481	goto out;
	482	}
	483
	484	efx_filter_table_clear_entry(efx, table, filter_idx);
	485	if (table->used == 0)
c39d35eb	486	efx_filter_table_reset_search_depth(table);
64eebcfd BH	487	rc = 0;
	488
	489	out:
	490	spin_unlock_bh(&state->lock);
	491	return rc;
	492	}
	493
8891681a BH	494	static void efx_filter_table_clear(struct efx_nic *efx,
	495	enum efx_filter_table_id table_id,
	496	enum efx_filter_priority priority)
64eebcfd BH	497	{
	498	struct efx_filter_state *state = efx->filter_state;
	499	struct efx_filter_table *table = &state->table[table_id];
	500	int filter_idx;
	501
	502	spin_lock_bh(&state->lock);
	503
	504	for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
	505	if (table->spec[filter_idx].priority <= priority)
	506	efx_filter_table_clear_entry(efx, table, filter_idx);
	507	if (table->used == 0)
c39d35eb	508	efx_filter_table_reset_search_depth(table);
64eebcfd BH	509
	510	spin_unlock_bh(&state->lock);
	511	}
	512
8891681a BH	513	/**
	514	* efx_filter_clear_rx - remove RX filters by priority
	515	* @efx: NIC from which to remove the filters
	516	* @priority: Maximum priority to remove
	517	*/
	518	void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
	519	{
	520	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
	521	efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
	522	}
	523
64eebcfd BH	524	/* Restore filter stater after reset */
	525	void efx_restore_filters(struct efx_nic *efx)
	526	{
	527	struct efx_filter_state *state = efx->filter_state;
	528	enum efx_filter_table_id table_id;
	529	struct efx_filter_table *table;
	530	efx_oword_t filter;
	531	int filter_idx;
	532
	533	spin_lock_bh(&state->lock);
	534
	535	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
	536	table = &state->table[table_id];
	537	for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
	538	if (!test_bit(filter_idx, table->used_bitmap))
	539	continue;
	540	efx_filter_build(&filter, &table->spec[filter_idx]);
	541	efx_writeo(efx, &filter,
	542	table->offset + table->step * filter_idx);
	543	}
	544	}
	545
	546	efx_filter_push_rx_limits(efx);
	547
	548	spin_unlock_bh(&state->lock);
	549	}
	550
	551	int efx_probe_filters(struct efx_nic *efx)
	552	{
	553	struct efx_filter_state *state;
	554	struct efx_filter_table *table;
	555	unsigned table_id;
	556
	557	state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
	558	if (!state)
	559	return -ENOMEM;
	560	efx->filter_state = state;
	561
	562	spin_lock_init(&state->lock);
	563
	564	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
	565	table = &state->table[EFX_FILTER_TABLE_RX_IP];
c39d35eb	566	table->id = EFX_FILTER_TABLE_RX_IP;
64eebcfd BH	567	table->offset = FR_BZ_RX_FILTER_TBL0;
	568	table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
	569	table->step = FR_BZ_RX_FILTER_TBL0_STEP;
	570	}
	571
	572	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
	573	table = &state->table[EFX_FILTER_TABLE_RX_MAC];
c39d35eb	574	table->id = EFX_FILTER_TABLE_RX_MAC;
64eebcfd BH	575	table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
	576	table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
	577	table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
	578	}
	579
	580	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
	581	table = &state->table[table_id];
	582	if (table->size == 0)
	583	continue;
	584	table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
	585	sizeof(unsigned long),
	586	GFP_KERNEL);
	587	if (!table->used_bitmap)
	588	goto fail;
89bf67f1	589	table->spec = vzalloc(table->size * sizeof(*table->spec));
64eebcfd BH	590	if (!table->spec)
64eebcfd BH	591	goto fail;
64eebcfd BH	592	}
	593
	594	return 0;
	595
	596	fail:
	597	efx_remove_filters(efx);
	598	return -ENOMEM;
	599	}
	600
	601	void efx_remove_filters(struct efx_nic *efx)
	602	{
	603	struct efx_filter_state *state = efx->filter_state;
	604	enum efx_filter_table_id table_id;
	605
	606	for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
	607	kfree(state->table[table_id].used_bitmap);
	608	vfree(state->table[table_id].spec);
	609	}
	610	kfree(state);
	611	}