[deliverable/linux.git] / net / rxrpc / ar-output.c

/* RxRPC packet transmission
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/net.h>
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/circ_buf.h>
#include <linux/export.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Time till packet resend (in jiffies).
 */
unsigned rxrpc_resend_timeout = 4 * HZ;

static int rxrpc_send_data(struct kiocb *iocb,
			   struct rxrpc_sock *rx,
			   struct rxrpc_call *call,
			   struct msghdr *msg, size_t len);

/*
 * extract control messages from the sendmsg() control buffer
 */
static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
			      unsigned long *user_call_ID,
			      enum rxrpc_command *command,
			      u32 *abort_code,
			      bool server)
{
	struct cmsghdr *cmsg;
	int len;

	*command = RXRPC_CMD_SEND_DATA;

	if (msg->msg_controllen == 0)
		return -EINVAL;

	for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
		if (!CMSG_OK(msg, cmsg))
			return -EINVAL;

		len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
		_debug("CMSG %d, %d, %d",
		       cmsg->cmsg_level, cmsg->cmsg_type, len);

		if (cmsg->cmsg_level != SOL_RXRPC)
			continue;

		switch (cmsg->cmsg_type) {
		case RXRPC_USER_CALL_ID:
			if (msg->msg_flags & MSG_CMSG_COMPAT) {
				if (len != sizeof(u32))
					return -EINVAL;
				*user_call_ID = *(u32 *) CMSG_DATA(cmsg);
			} else {
				if (len != sizeof(unsigned long))
					return -EINVAL;
				*user_call_ID = *(unsigned long *)
					CMSG_DATA(cmsg);
			}
			_debug("User Call ID %lx", *user_call_ID);
			break;

		case RXRPC_ABORT:
			if (*command != RXRPC_CMD_SEND_DATA)
				return -EINVAL;
			*command = RXRPC_CMD_SEND_ABORT;
			if (len != sizeof(*abort_code))
				return -EINVAL;
			*abort_code = *(unsigned int *) CMSG_DATA(cmsg);
			_debug("Abort %x", *abort_code);
			if (*abort_code == 0)
				return -EINVAL;
			break;

		case RXRPC_ACCEPT:
			if (*command != RXRPC_CMD_SEND_DATA)
				return -EINVAL;
			*command = RXRPC_CMD_ACCEPT;
			if (len != 0)
				return -EINVAL;
			if (!server)
				return -EISCONN;
			break;

		default:
			return -EINVAL;
		}
	}

	_leave(" = 0");
	return 0;
}

/*
 * abort a call, sending an ABORT packet to the peer
 */
static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
{
	write_lock_bh(&call->state_lock);

	if (call->state <= RXRPC_CALL_COMPLETE) {
		call->state = RXRPC_CALL_LOCALLY_ABORTED;
		call->abort_code = abort_code;
		set_bit(RXRPC_CALL_ABORT, &call->events);
		del_timer_sync(&call->resend_timer);
		del_timer_sync(&call->ack_timer);
		clear_bit(RXRPC_CALL_RESEND_TIMER, &call->events);
		clear_bit(RXRPC_CALL_ACK, &call->events);
		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
		rxrpc_queue_call(call);
	}

	write_unlock_bh(&call->state_lock);
}

/*
 * send a message forming part of a client call through an RxRPC socket
 * - caller holds the socket locked
 * - the socket may be either a client socket or a server socket
 */
int rxrpc_client_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
			 struct rxrpc_transport *trans, struct msghdr *msg,
			 size_t len)
{
	struct rxrpc_conn_bundle *bundle;
	enum rxrpc_command cmd;
	struct rxrpc_call *call;
	unsigned long user_call_ID = 0;
	struct key *key;
	__be16 service_id;
	u32 abort_code = 0;
	int ret;

	_enter("");

	ASSERT(trans != NULL);

	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
				 false);
	if (ret < 0)
		return ret;

	bundle = NULL;
	if (trans) {
		service_id = rx->service_id;
		if (msg->msg_name) {
			DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
					 msg->msg_name);
			service_id = htons(srx->srx_service);
		}
		key = rx->key;
		if (key && !rx->key->payload.data)
			key = NULL;
		bundle = rxrpc_get_bundle(rx, trans, key, service_id,
					  GFP_KERNEL);
		if (IS_ERR(bundle))
			return PTR_ERR(bundle);
	}

	call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
				     abort_code == 0, GFP_KERNEL);
	if (trans)
		rxrpc_put_bundle(trans, bundle);
	if (IS_ERR(call)) {
		_leave(" = %ld", PTR_ERR(call));
		return PTR_ERR(call);
	}

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state >= RXRPC_CALL_COMPLETE) {
		/* it's too late for this call */
		ret = -ESHUTDOWN;
	} else if (cmd == RXRPC_CMD_SEND_ABORT) {
		rxrpc_send_abort(call, abort_code);
	} else if (cmd != RXRPC_CMD_SEND_DATA) {
		ret = -EINVAL;
	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
		/* request phase complete for this client call */
		ret = -EPROTO;
	} else {
		ret = rxrpc_send_data(iocb, rx, call, msg, len);
	}

	rxrpc_put_call(call);
	_leave(" = %d", ret);
	return ret;
}

/**
 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 * @call: The call to send data through
 * @msg: The data to send
 * @len: The amount of data to send
 *
 * Allow a kernel service to send data on a call.  The call must be in an state
 * appropriate to sending data.  No control data should be supplied in @msg,
 * nor should an address be supplied.  MSG_MORE should be flagged if there's
 * more data to come, otherwise this data will end the transmission phase.
 */
int rxrpc_kernel_send_data(struct rxrpc_call *call, struct msghdr *msg,
			   size_t len)
{
	int ret;

	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);

	ASSERTCMP(msg->msg_name, ==, NULL);
	ASSERTCMP(msg->msg_control, ==, NULL);

	lock_sock(&call->socket->sk);

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state >= RXRPC_CALL_COMPLETE) {
		ret = -ESHUTDOWN; /* it's too late for this call */
	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
		   call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
		   call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
		ret = -EPROTO; /* request phase complete for this client call */
	} else {
		mm_segment_t oldfs = get_fs();
		set_fs(KERNEL_DS);
		ret = rxrpc_send_data(NULL, call->socket, call, msg, len);
		set_fs(oldfs);
	}

	release_sock(&call->socket->sk);
	_leave(" = %d", ret);
	return ret;
}

EXPORT_SYMBOL(rxrpc_kernel_send_data);

/**
 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 * @call: The call to be aborted
 * @abort_code: The abort code to stick into the ABORT packet
 *
 * Allow a kernel service to abort a call, if it's still in an abortable state.
 */
void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
{
	_enter("{%d},%d", call->debug_id, abort_code);

	lock_sock(&call->socket->sk);

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state < RXRPC_CALL_COMPLETE)
		rxrpc_send_abort(call, abort_code);

	release_sock(&call->socket->sk);
	_leave("");
}

EXPORT_SYMBOL(rxrpc_kernel_abort_call);

/*
 * send a message through a server socket
 * - caller holds the socket locked
 */
int rxrpc_server_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
			 struct msghdr *msg, size_t len)
{
	enum rxrpc_command cmd;
	struct rxrpc_call *call;
	unsigned long user_call_ID = 0;
	u32 abort_code = 0;
	int ret;

	_enter("");

	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
				 true);
	if (ret < 0)
		return ret;

	if (cmd == RXRPC_CMD_ACCEPT) {
		call = rxrpc_accept_call(rx, user_call_ID);
		if (IS_ERR(call))
			return PTR_ERR(call);
		rxrpc_put_call(call);
		return 0;
	}

	call = rxrpc_find_server_call(rx, user_call_ID);
	if (!call)
		return -EBADSLT;
	if (call->state >= RXRPC_CALL_COMPLETE) {
		ret = -ESHUTDOWN;
		goto out;
	}

	switch (cmd) {
	case RXRPC_CMD_SEND_DATA:
		if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
		    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
		    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
			/* Tx phase not yet begun for this call */
			ret = -EPROTO;
			break;
		}

		ret = rxrpc_send_data(iocb, rx, call, msg, len);
		break;

	case RXRPC_CMD_SEND_ABORT:
		rxrpc_send_abort(call, abort_code);
		break;
	default:
		BUG();
	}

	out:
	rxrpc_put_call(call);
	_leave(" = %d", ret);
	return ret;
}

/*
 * send a packet through the transport endpoint
 */
int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
{
	struct kvec iov[1];
	struct msghdr msg;
	int ret, opt;

	_enter(",{%d}", skb->len);

	iov[0].iov_base = skb->head;
	iov[0].iov_len = skb->len;

	msg.msg_name = &trans->peer->srx.transport.sin;
	msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags = 0;

	/* send the packet with the don't fragment bit set if we currently
	 * think it's small enough */
	if (skb->len - sizeof(struct rxrpc_header) < trans->peer->maxdata) {
		down_read(&trans->local->defrag_sem);
		/* send the packet by UDP
		 * - returns -EMSGSIZE if UDP would have to fragment the packet
		 *   to go out of the interface
		 *   - in which case, we'll have processed the ICMP error
		 *     message and update the peer record
		 */
		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
				     iov[0].iov_len);

		up_read(&trans->local->defrag_sem);
		if (ret == -EMSGSIZE)
			goto send_fragmentable;

		_leave(" = %d [%u]", ret, trans->peer->maxdata);
		return ret;
	}

send_fragmentable:
	/* attempt to send this message with fragmentation enabled */
	_debug("send fragment");

	down_write(&trans->local->defrag_sem);
	opt = IP_PMTUDISC_DONT;
	ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
				(char *) &opt, sizeof(opt));
	if (ret == 0) {
		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
				     iov[0].iov_len);

		opt = IP_PMTUDISC_DO;
		kernel_setsockopt(trans->local->socket, SOL_IP,
				  IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
	}

	up_write(&trans->local->defrag_sem);
	_leave(" = %d [frag %u]", ret, trans->peer->maxdata);
	return ret;
}

/*
 * wait for space to appear in the transmit/ACK window
 * - caller holds the socket locked
 */
static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
				    struct rxrpc_call *call,
				    long *timeo)
{
	DECLARE_WAITQUEUE(myself, current);
	int ret;

	_enter(",{%d},%ld",
	       CIRC_SPACE(call->acks_head, call->acks_tail, call->acks_winsz),
	       *timeo);

	add_wait_queue(&call->tx_waitq, &myself);

	for (;;) {
		set_current_state(TASK_INTERRUPTIBLE);
		ret = 0;
		if (CIRC_SPACE(call->acks_head, call->acks_tail,
			       call->acks_winsz) > 0)
			break;
		if (signal_pending(current)) {
			ret = sock_intr_errno(*timeo);
			break;
		}

		release_sock(&rx->sk);
		*timeo = schedule_timeout(*timeo);
		lock_sock(&rx->sk);
	}

	remove_wait_queue(&call->tx_waitq, &myself);
	set_current_state(TASK_RUNNING);
	_leave(" = %d", ret);
	return ret;
}

/*
 * attempt to schedule an instant Tx resend
 */
static inline void rxrpc_instant_resend(struct rxrpc_call *call)
{
	read_lock_bh(&call->state_lock);
	if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
		if (call->state < RXRPC_CALL_COMPLETE &&
		    !test_and_set_bit(RXRPC_CALL_RESEND_TIMER, &call->events))
			rxrpc_queue_call(call);
	}
	read_unlock_bh(&call->state_lock);
}

/*
 * queue a packet for transmission, set the resend timer and attempt
 * to send the packet immediately
 */
static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
			       bool last)
{
	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	int ret;

	_net("queue skb %p [%d]", skb, call->acks_head);

	ASSERT(call->acks_window != NULL);
	call->acks_window[call->acks_head] = (unsigned long) skb;
	smp_wmb();
	call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);

	if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
		_debug("________awaiting reply/ACK__________");
		write_lock_bh(&call->state_lock);
		switch (call->state) {
		case RXRPC_CALL_CLIENT_SEND_REQUEST:
			call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
			break;
		case RXRPC_CALL_SERVER_ACK_REQUEST:
			call->state = RXRPC_CALL_SERVER_SEND_REPLY;
			if (!last)
				break;
		case RXRPC_CALL_SERVER_SEND_REPLY:
			call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
			break;
		default:
			break;
		}
		write_unlock_bh(&call->state_lock);
	}

	_proto("Tx DATA %%%u { #%u }",
	       ntohl(sp->hdr.serial), ntohl(sp->hdr.seq));

	sp->need_resend = false;
	sp->resend_at = jiffies + rxrpc_resend_timeout;
	if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
		_debug("run timer");
		call->resend_timer.expires = sp->resend_at;
		add_timer(&call->resend_timer);
	}

	/* attempt to cancel the rx-ACK timer, deferring reply transmission if
	 * we're ACK'ing the request phase of an incoming call */
	ret = -EAGAIN;
	if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
		/* the packet may be freed by rxrpc_process_call() before this
		 * returns */
		ret = rxrpc_send_packet(call->conn->trans, skb);
		_net("sent skb %p", skb);
	} else {
		_debug("failed to delete ACK timer");
	}

	if (ret < 0) {
		_debug("need instant resend %d", ret);
		sp->need_resend = true;
		rxrpc_instant_resend(call);
	}

	_leave("");
}

/*
 * send data through a socket
 * - must be called in process context
 * - caller holds the socket locked
 */
static int rxrpc_send_data(struct kiocb *iocb,
			   struct rxrpc_sock *rx,
			   struct rxrpc_call *call,
			   struct msghdr *msg, size_t len)
{
	struct rxrpc_skb_priv *sp;
	unsigned char __user *from;
	struct sk_buff *skb;
	struct iovec *iov;
	struct sock *sk = &rx->sk;
	long timeo;
	bool more;
	int ret, ioc, segment, copied;

	_enter(",,,{%zu},%zu", msg->msg_iovlen, len);

	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

	/* this should be in poll */
	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);

	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
		return -EPIPE;

	iov = msg->msg_iov;
	ioc = msg->msg_iovlen - 1;
	from = iov->iov_base;
	segment = iov->iov_len;
	iov++;
	more = msg->msg_flags & MSG_MORE;

	skb = call->tx_pending;
	call->tx_pending = NULL;

	copied = 0;
	do {
		int copy;

		if (segment > len)
			segment = len;

		_debug("SEGMENT %d @%p", segment, from);

		if (!skb) {
			size_t size, chunk, max, space;

			_debug("alloc");

			if (CIRC_SPACE(call->acks_head, call->acks_tail,
				       call->acks_winsz) <= 0) {
				ret = -EAGAIN;
				if (msg->msg_flags & MSG_DONTWAIT)
					goto maybe_error;
				ret = rxrpc_wait_for_tx_window(rx, call,
							       &timeo);
				if (ret < 0)
					goto maybe_error;
			}

			max = call->conn->trans->peer->maxdata;
			max -= call->conn->security_size;
			max &= ~(call->conn->size_align - 1UL);

			chunk = max;
			if (chunk > len && !more)
				chunk = len;

			space = chunk + call->conn->size_align;
			space &= ~(call->conn->size_align - 1UL);

			size = space + call->conn->header_size;

			_debug("SIZE: %zu/%zu/%zu", chunk, space, size);

			/* create a buffer that we can retain until it's ACK'd */
			skb = sock_alloc_send_skb(
				sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
			if (!skb)
				goto maybe_error;

			rxrpc_new_skb(skb);

			_debug("ALLOC SEND %p", skb);

			ASSERTCMP(skb->mark, ==, 0);

			_debug("HS: %u", call->conn->header_size);
			skb_reserve(skb, call->conn->header_size);
			skb->len += call->conn->header_size;

			sp = rxrpc_skb(skb);
			sp->remain = chunk;
			if (sp->remain > skb_tailroom(skb))
				sp->remain = skb_tailroom(skb);

			_net("skb: hr %d, tr %d, hl %d, rm %d",
			       skb_headroom(skb),
			       skb_tailroom(skb),
			       skb_headlen(skb),
			       sp->remain);

			skb->ip_summed = CHECKSUM_UNNECESSARY;
		}

		_debug("append");
		sp = rxrpc_skb(skb);

		/* append next segment of data to the current buffer */
		copy = skb_tailroom(skb);
		ASSERTCMP(copy, >, 0);
		if (copy > segment)
			copy = segment;
		if (copy > sp->remain)
			copy = sp->remain;

		_debug("add");
		ret = skb_add_data(skb, from, copy);
		_debug("added");
		if (ret < 0)
			goto efault;
		sp->remain -= copy;
		skb->mark += copy;
		copied += copy;

		len -= copy;
		segment -= copy;
		from += copy;
		while (segment == 0 && ioc > 0) {
			from = iov->iov_base;
			segment = iov->iov_len;
			iov++;
			ioc--;
		}
		if (len == 0) {
			segment = 0;
			ioc = 0;
		}

		/* check for the far side aborting the call or a network error
		 * occurring */
		if (call->state > RXRPC_CALL_COMPLETE)
			goto call_aborted;

		/* add the packet to the send queue if it's now full */
		if (sp->remain <= 0 || (segment == 0 && !more)) {
			struct rxrpc_connection *conn = call->conn;
			uint32_t seq;
			size_t pad;

			/* pad out if we're using security */
			if (conn->security) {
				pad = conn->security_size + skb->mark;
				pad = conn->size_align - pad;
				pad &= conn->size_align - 1;
				_debug("pad %zu", pad);
				if (pad)
					memset(skb_put(skb, pad), 0, pad);
			}

			seq = atomic_inc_return(&call->sequence);

			sp->hdr.epoch = conn->epoch;
			sp->hdr.cid = call->cid;
			sp->hdr.callNumber = call->call_id;
			sp->hdr.seq = htonl(seq);
			sp->hdr.serial =
				htonl(atomic_inc_return(&conn->serial));
			sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
			sp->hdr.userStatus = 0;
			sp->hdr.securityIndex = conn->security_ix;
			sp->hdr._rsvd = 0;
			sp->hdr.serviceId = conn->service_id;

			sp->hdr.flags = conn->out_clientflag;
			if (len == 0 && !more)
				sp->hdr.flags |= RXRPC_LAST_PACKET;
			else if (CIRC_SPACE(call->acks_head, call->acks_tail,
					    call->acks_winsz) > 1)
				sp->hdr.flags |= RXRPC_MORE_PACKETS;
			if (more && seq & 1)
				sp->hdr.flags |= RXRPC_REQUEST_ACK;

			ret = rxrpc_secure_packet(
				call, skb, skb->mark,
				skb->head + sizeof(struct rxrpc_header));
			if (ret < 0)
				goto out;

			memcpy(skb->head, &sp->hdr,
			       sizeof(struct rxrpc_header));
			rxrpc_queue_packet(call, skb, segment == 0 && !more);
			skb = NULL;
		}

	} while (segment > 0);

success:
	ret = copied;
out:
	call->tx_pending = skb;
	_leave(" = %d", ret);
	return ret;

call_aborted:
	rxrpc_free_skb(skb);
	if (call->state == RXRPC_CALL_NETWORK_ERROR)
		ret = call->conn->trans->peer->net_error;
	else
		ret = -ECONNABORTED;
	_leave(" = %d", ret);
	return ret;

maybe_error:
	if (copied)
		goto success;
	goto out;

efault:
	ret = -EFAULT;
	goto out;
}
Commit	Line	Data
17926a79 DH	1	/* RxRPC packet transmission
	2	*
	3	* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
	4	* Written by David Howells (dhowells@redhat.com)
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/net.h>
5a0e3ad6	13	#include <linux/gfp.h>
17926a79 DH	14	#include <linux/skbuff.h>
17926a79 DH	15	#include <linux/circ_buf.h>
bc3b2d7f	16	#include <linux/export.h>
17926a79 DH	17	#include <net/sock.h>
	18	#include <net/af_rxrpc.h>
	19	#include "ar-internal.h"
	20
5873c083 DH	21	/*
	22	* Time till packet resend (in jiffies).
	23	*/
	24	unsigned rxrpc_resend_timeout = 4 * HZ;
17926a79 DH	25
	26	static int rxrpc_send_data(struct kiocb *iocb,
	27	struct rxrpc_sock *rx,
	28	struct rxrpc_call *call,
	29	struct msghdr *msg, size_t len);
	30
	31	/*
	32	* extract control messages from the sendmsg() control buffer
	33	*/
	34	static int rxrpc_sendmsg_cmsg(struct rxrpc_sock rx, struct msghdr msg,
	35	unsigned long *user_call_ID,
	36	enum rxrpc_command *command,
	37	u32 *abort_code,
	38	bool server)
	39	{
	40	struct cmsghdr *cmsg;
	41	int len;
	42
	43	*command = RXRPC_CMD_SEND_DATA;
	44
	45	if (msg->msg_controllen == 0)
	46	return -EINVAL;
	47
	48	for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
	49	if (!CMSG_OK(msg, cmsg))
	50	return -EINVAL;
	51
	52	len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
	53	_debug("CMSG %d, %d, %d",
	54	cmsg->cmsg_level, cmsg->cmsg_type, len);
	55
	56	if (cmsg->cmsg_level != SOL_RXRPC)
	57	continue;
	58
	59	switch (cmsg->cmsg_type) {
	60	case RXRPC_USER_CALL_ID:
	61	if (msg->msg_flags & MSG_CMSG_COMPAT) {
	62	if (len != sizeof(u32))
	63	return -EINVAL;
	64	user_call_ID = (u32 *) CMSG_DATA(cmsg);
	65	} else {
	66	if (len != sizeof(unsigned long))
	67	return -EINVAL;
	68	user_call_ID = (unsigned long *)
	69	CMSG_DATA(cmsg);
	70	}
	71	_debug("User Call ID %lx", *user_call_ID);
	72	break;
	73
	74	case RXRPC_ABORT:
	75	if (*command != RXRPC_CMD_SEND_DATA)
	76	return -EINVAL;
	77	*command = RXRPC_CMD_SEND_ABORT;
	78	if (len != sizeof(*abort_code))
	79	return -EINVAL;
	80	abort_code = (unsigned int *) CMSG_DATA(cmsg);
	81	_debug("Abort %x", *abort_code);
	82	if (*abort_code == 0)
	83	return -EINVAL;
	84	break;
	85
	86	case RXRPC_ACCEPT:
	87	if (*command != RXRPC_CMD_SEND_DATA)
	88	return -EINVAL;
89	*command = RXRPC_CMD_ACCEPT;
90	if (len != 0)
91	return -EINVAL;
92	if (!server)
93	return -EISCONN;
94	break;
95
96	default:
97	return -EINVAL;
98	}
99	}
100
101	_leave(" = 0");
102	return 0;
103	}
104
105	/*
106	* abort a call, sending an ABORT packet to the peer
107	*/
108	static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
109	{
110	write_lock_bh(&call->state_lock);
111
112	if (call->state <= RXRPC_CALL_COMPLETE) {
113	call->state = RXRPC_CALL_LOCALLY_ABORTED;
114	call->abort_code = abort_code;
115	set_bit(RXRPC_CALL_ABORT, &call->events);
116	del_timer_sync(&call->resend_timer);
117	del_timer_sync(&call->ack_timer);
118	clear_bit(RXRPC_CALL_RESEND_TIMER, &call->events);
119	clear_bit(RXRPC_CALL_ACK, &call->events);
120	clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
651350d1	121	rxrpc_queue_call(call);
17926a79 DH	122	}
	123
	124	write_unlock_bh(&call->state_lock);
	125	}
	126
	127	/*
	128	* send a message forming part of a client call through an RxRPC socket
	129	* - caller holds the socket locked
	130	* - the socket may be either a client socket or a server socket
	131	*/
	132	int rxrpc_client_sendmsg(struct kiocb iocb, struct rxrpc_sock rx,
	133	struct rxrpc_transport trans, struct msghdr msg,
	134	size_t len)
	135	{
	136	struct rxrpc_conn_bundle *bundle;
	137	enum rxrpc_command cmd;
	138	struct rxrpc_call *call;
	139	unsigned long user_call_ID = 0;
	140	struct key *key;
	141	__be16 service_id;
	142	u32 abort_code = 0;
	143	int ret;
	144
	145	_enter("");
	146
	147	ASSERT(trans != NULL);
	148
	149	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
	150	false);
	151	if (ret < 0)
	152	return ret;
	153
	154	bundle = NULL;
	155	if (trans) {
	156	service_id = rx->service_id;
	157	if (msg->msg_name) {
342dfc30 SH	158	DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
342dfc30 SH	159	msg->msg_name);
17926a79 DH	160	service_id = htons(srx->srx_service);
	161	}
	162	key = rx->key;
	163	if (key && !rx->key->payload.data)
	164	key = NULL;
	165	bundle = rxrpc_get_bundle(rx, trans, key, service_id,
	166	GFP_KERNEL);
	167	if (IS_ERR(bundle))
	168	return PTR_ERR(bundle);
	169	}
	170
	171	call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
	172	abort_code == 0, GFP_KERNEL);
	173	if (trans)
	174	rxrpc_put_bundle(trans, bundle);
	175	if (IS_ERR(call)) {
	176	_leave(" = %ld", PTR_ERR(call));
	177	return PTR_ERR(call);
	178	}
	179
	180	_debug("CALL %d USR %lx ST %d on CONN %p",
	181	call->debug_id, call->user_call_ID, call->state, call->conn);
	182
	183	if (call->state >= RXRPC_CALL_COMPLETE) {
	184	/* it's too late for this call */
	185	ret = -ESHUTDOWN;
	186	} else if (cmd == RXRPC_CMD_SEND_ABORT) {
	187	rxrpc_send_abort(call, abort_code);
	188	} else if (cmd != RXRPC_CMD_SEND_DATA) {
	189	ret = -EINVAL;
	190	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
	191	/* request phase complete for this client call */
	192	ret = -EPROTO;
	193	} else {
	194	ret = rxrpc_send_data(iocb, rx, call, msg, len);
	195	}
	196
	197	rxrpc_put_call(call);
	198	_leave(" = %d", ret);
	199	return ret;
	200	}
	201
651350d1 DH	202	/**
	203	* rxrpc_kernel_send_data - Allow a kernel service to send data on a call
	204	* @call: The call to send data through
	205	* @msg: The data to send
	206	* @len: The amount of data to send
	207	*
	208	* Allow a kernel service to send data on a call. The call must be in an state
	209	* appropriate to sending data. No control data should be supplied in @msg,
	210	* nor should an address be supplied. MSG_MORE should be flagged if there's
	211	* more data to come, otherwise this data will end the transmission phase.
	212	*/
	213	int rxrpc_kernel_send_data(struct rxrpc_call call, struct msghdr msg,
	214	size_t len)
	215	{
	216	int ret;
	217
	218	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);
	219
	220	ASSERTCMP(msg->msg_name, ==, NULL);
	221	ASSERTCMP(msg->msg_control, ==, NULL);
	222
	223	lock_sock(&call->socket->sk);
	224
	225	_debug("CALL %d USR %lx ST %d on CONN %p",
	226	call->debug_id, call->user_call_ID, call->state, call->conn);
	227
	228	if (call->state >= RXRPC_CALL_COMPLETE) {
	229	ret = -ESHUTDOWN; /* it's too late for this call */
	230	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
	231	call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
	232	call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
	233	ret = -EPROTO; /* request phase complete for this client call */
	234	} else {
	235	mm_segment_t oldfs = get_fs();
	236	set_fs(KERNEL_DS);
	237	ret = rxrpc_send_data(NULL, call->socket, call, msg, len);
	238	set_fs(oldfs);
	239	}
	240
	241	release_sock(&call->socket->sk);
	242	_leave(" = %d", ret);
	243	return ret;
	244	}
	245
	246	EXPORT_SYMBOL(rxrpc_kernel_send_data);
	247
2c53040f	248	/**
651350d1 DH	249	* rxrpc_kernel_abort_call - Allow a kernel service to abort a call
	250	* @call: The call to be aborted
	251	* @abort_code: The abort code to stick into the ABORT packet
	252	*
	253	* Allow a kernel service to abort a call, if it's still in an abortable state.
	254	*/
	255	void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
	256	{
	257	_enter("{%d},%d", call->debug_id, abort_code);
	258
	259	lock_sock(&call->socket->sk);
	260
	261	_debug("CALL %d USR %lx ST %d on CONN %p",
	262	call->debug_id, call->user_call_ID, call->state, call->conn);
	263
	264	if (call->state < RXRPC_CALL_COMPLETE)
	265	rxrpc_send_abort(call, abort_code);
	266
	267	release_sock(&call->socket->sk);
	268	_leave("");
	269	}
	270
	271	EXPORT_SYMBOL(rxrpc_kernel_abort_call);
	272
17926a79 DH	273	/*
	274	* send a message through a server socket
	275	* - caller holds the socket locked
	276	*/
	277	int rxrpc_server_sendmsg(struct kiocb iocb, struct rxrpc_sock rx,
	278	struct msghdr *msg, size_t len)
	279	{
	280	enum rxrpc_command cmd;
	281	struct rxrpc_call *call;
	282	unsigned long user_call_ID = 0;
	283	u32 abort_code = 0;
	284	int ret;
	285
	286	_enter("");
	287
	288	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
	289	true);
	290	if (ret < 0)
	291	return ret;
	292
651350d1 DH	293	if (cmd == RXRPC_CMD_ACCEPT) {
	294	call = rxrpc_accept_call(rx, user_call_ID);
	295	if (IS_ERR(call))
	296	return PTR_ERR(call);
	297	rxrpc_put_call(call);
	298	return 0;
	299	}
17926a79 DH	300
	301	call = rxrpc_find_server_call(rx, user_call_ID);
	302	if (!call)
	303	return -EBADSLT;
	304	if (call->state >= RXRPC_CALL_COMPLETE) {
	305	ret = -ESHUTDOWN;
	306	goto out;
	307	}
	308
	309	switch (cmd) {
	310	case RXRPC_CMD_SEND_DATA:
	311	if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
	312	call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
	313	call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
	314	/* Tx phase not yet begun for this call */
	315	ret = -EPROTO;
	316	break;
	317	}
	318
	319	ret = rxrpc_send_data(iocb, rx, call, msg, len);
	320	break;
	321
	322	case RXRPC_CMD_SEND_ABORT:
	323	rxrpc_send_abort(call, abort_code);
	324	break;
	325	default:
	326	BUG();
	327	}
	328
	329	out:
	330	rxrpc_put_call(call);
	331	_leave(" = %d", ret);
	332	return ret;
	333	}
	334
	335	/*
	336	* send a packet through the transport endpoint
	337	*/
	338	int rxrpc_send_packet(struct rxrpc_transport trans, struct sk_buff skb)
	339	{
	340	struct kvec iov[1];
	341	struct msghdr msg;
	342	int ret, opt;
	343
	344	_enter(",{%d}", skb->len);
	345
	346	iov[0].iov_base = skb->head;
	347	iov[0].iov_len = skb->len;
	348
	349	msg.msg_name = &trans->peer->srx.transport.sin;
	350	msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
	351	msg.msg_control = NULL;
	352	msg.msg_controllen = 0;
	353	msg.msg_flags = 0;
	354
	355	/* send the packet with the don't fragment bit set if we currently
	356	* think it's small enough */
	357	if (skb->len - sizeof(struct rxrpc_header) < trans->peer->maxdata) {
	358	down_read(&trans->local->defrag_sem);
	359	/* send the packet by UDP
	360	* - returns -EMSGSIZE if UDP would have to fragment the packet
	361	* to go out of the interface
	362	* - in which case, we'll have processed the ICMP error
	363	* message and update the peer record
364	*/
365	ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
366	iov[0].iov_len);
367
368	up_read(&trans->local->defrag_sem);
369	if (ret == -EMSGSIZE)
370	goto send_fragmentable;
371
372	_leave(" = %d [%u]", ret, trans->peer->maxdata);
373	return ret;
374	}
375
376	send_fragmentable:
377	/* attempt to send this message with fragmentation enabled */
378	_debug("send fragment");
379
380	down_write(&trans->local->defrag_sem);
381	opt = IP_PMTUDISC_DONT;
382	ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
383	(char *) &opt, sizeof(opt));
384	if (ret == 0) {
385	ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
386	iov[0].iov_len);
387
388	opt = IP_PMTUDISC_DO;
389	kernel_setsockopt(trans->local->socket, SOL_IP,
390	IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
391	}
392
393	up_write(&trans->local->defrag_sem);
394	_leave(" = %d [frag %u]", ret, trans->peer->maxdata);
395	return ret;
396	}
397
398	/*
399	* wait for space to appear in the transmit/ACK window
400	* - caller holds the socket locked
401	*/
402	static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
403	struct rxrpc_call *call,
404	long *timeo)
405	{
406	DECLARE_WAITQUEUE(myself, current);
407	int ret;
408
409	_enter(",{%d},%ld",
410	CIRC_SPACE(call->acks_head, call->acks_tail, call->acks_winsz),
411	*timeo);
412
413	add_wait_queue(&call->tx_waitq, &myself);
414
415	for (;;) {
416	set_current_state(TASK_INTERRUPTIBLE);
417	ret = 0;
418	if (CIRC_SPACE(call->acks_head, call->acks_tail,
419	call->acks_winsz) > 0)
420	break;
421	if (signal_pending(current)) {
422	ret = sock_intr_errno(*timeo);
423	break;
424	}
425
426	release_sock(&rx->sk);
427	timeo = schedule_timeout(timeo);
428	lock_sock(&rx->sk);
429	}
430
431	remove_wait_queue(&call->tx_waitq, &myself);
432	set_current_state(TASK_RUNNING);
433	_leave(" = %d", ret);
434	return ret;
435	}
436
437	/*
438	* attempt to schedule an instant Tx resend
439	*/
440	static inline void rxrpc_instant_resend(struct rxrpc_call *call)
441	{
442	read_lock_bh(&call->state_lock);
443	if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
444	clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
445	if (call->state < RXRPC_CALL_COMPLETE &&
446	!test_and_set_bit(RXRPC_CALL_RESEND_TIMER, &call->events))
651350d1	447	rxrpc_queue_call(call);
17926a79 DH	448	}
	449	read_unlock_bh(&call->state_lock);
	450	}
	451
	452	/*
	453	* queue a packet for transmission, set the resend timer and attempt
	454	* to send the packet immediately
	455	*/
	456	static void rxrpc_queue_packet(struct rxrpc_call call, struct sk_buff skb,
	457	bool last)
	458	{
	459	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	460	int ret;
	461
	462	_net("queue skb %p [%d]", skb, call->acks_head);
	463
	464	ASSERT(call->acks_window != NULL);
	465	call->acks_window[call->acks_head] = (unsigned long) skb;
	466	smp_wmb();
	467	call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);
	468
	469	if (last \|\| call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
	470	_debug("________awaiting reply/ACK__________");
	471	write_lock_bh(&call->state_lock);
	472	switch (call->state) {
	473	case RXRPC_CALL_CLIENT_SEND_REQUEST:
	474	call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
	475	break;
	476	case RXRPC_CALL_SERVER_ACK_REQUEST:
	477	call->state = RXRPC_CALL_SERVER_SEND_REPLY;
	478	if (!last)
	479	break;
	480	case RXRPC_CALL_SERVER_SEND_REPLY:
	481	call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
	482	break;
	483	default:
	484	break;
	485	}
	486	write_unlock_bh(&call->state_lock);
	487	}
	488
	489	_proto("Tx DATA %%%u { #%u }",
	490	ntohl(sp->hdr.serial), ntohl(sp->hdr.seq));
	491
3db1cd5c	492	sp->need_resend = false;
5873c083	493	sp->resend_at = jiffies + rxrpc_resend_timeout;
17926a79 DH	494	if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
	495	_debug("run timer");
	496	call->resend_timer.expires = sp->resend_at;
	497	add_timer(&call->resend_timer);
	498	}
	499
	500	/* attempt to cancel the rx-ACK timer, deferring reply transmission if
	501	* we're ACK'ing the request phase of an incoming call */
	502	ret = -EAGAIN;
	503	if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
	504	/* the packet may be freed by rxrpc_process_call() before this
	505	* returns */
	506	ret = rxrpc_send_packet(call->conn->trans, skb);
	507	_net("sent skb %p", skb);
	508	} else {
	509	_debug("failed to delete ACK timer");
	510	}
	511
	512	if (ret < 0) {
	513	_debug("need instant resend %d", ret);
3db1cd5c	514	sp->need_resend = true;
17926a79 DH	515	rxrpc_instant_resend(call);
	516	}
	517
	518	_leave("");
	519	}
	520
	521	/*
	522	* send data through a socket
	523	* - must be called in process context
	524	* - caller holds the socket locked
	525	*/
	526	static int rxrpc_send_data(struct kiocb *iocb,
	527	struct rxrpc_sock *rx,
	528	struct rxrpc_call *call,
	529	struct msghdr *msg, size_t len)
	530	{
	531	struct rxrpc_skb_priv *sp;
	532	unsigned char __user *from;
	533	struct sk_buff *skb;
	534	struct iovec *iov;
	535	struct sock *sk = &rx->sk;
	536	long timeo;
	537	bool more;
	538	int ret, ioc, segment, copied;
	539
	540	_enter(",,,{%zu},%zu", msg->msg_iovlen, len);
	541
	542	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
	543
	544	/* this should be in poll */
	545	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
	546
	547	if (sk->sk_err \|\| (sk->sk_shutdown & SEND_SHUTDOWN))
	548	return -EPIPE;
	549
	550	iov = msg->msg_iov;
	551	ioc = msg->msg_iovlen - 1;
	552	from = iov->iov_base;
	553	segment = iov->iov_len;
	554	iov++;
	555	more = msg->msg_flags & MSG_MORE;
	556
	557	skb = call->tx_pending;
	558	call->tx_pending = NULL;
	559
	560	copied = 0;
	561	do {
	562	int copy;
	563
	564	if (segment > len)
	565	segment = len;
	566
	567	_debug("SEGMENT %d @%p", segment, from);
	568
	569	if (!skb) {
	570	size_t size, chunk, max, space;
	571
	572	_debug("alloc");
	573
	574	if (CIRC_SPACE(call->acks_head, call->acks_tail,
	575	call->acks_winsz) <= 0) {
	576	ret = -EAGAIN;
	577	if (msg->msg_flags & MSG_DONTWAIT)
	578	goto maybe_error;
579	ret = rxrpc_wait_for_tx_window(rx, call,
580	&timeo);
581	if (ret < 0)
582	goto maybe_error;
583	}
584
585	max = call->conn->trans->peer->maxdata;
586	max -= call->conn->security_size;
587	max &= ~(call->conn->size_align - 1UL);
588
589	chunk = max;
224711df	590	if (chunk > len && !more)
17926a79 DH	591	chunk = len;
	592
	593	space = chunk + call->conn->size_align;
	594	space &= ~(call->conn->size_align - 1UL);
	595
	596	size = space + call->conn->header_size;
	597
	598	_debug("SIZE: %zu/%zu/%zu", chunk, space, size);
	599
	600	/* create a buffer that we can retain until it's ACK'd */
	601	skb = sock_alloc_send_skb(
	602	sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
	603	if (!skb)
	604	goto maybe_error;
	605
	606	rxrpc_new_skb(skb);
	607
	608	_debug("ALLOC SEND %p", skb);
	609
	610	ASSERTCMP(skb->mark, ==, 0);
	611
	612	_debug("HS: %u", call->conn->header_size);
	613	skb_reserve(skb, call->conn->header_size);
	614	skb->len += call->conn->header_size;
	615
	616	sp = rxrpc_skb(skb);
	617	sp->remain = chunk;
	618	if (sp->remain > skb_tailroom(skb))
	619	sp->remain = skb_tailroom(skb);
	620
	621	_net("skb: hr %d, tr %d, hl %d, rm %d",
	622	skb_headroom(skb),
	623	skb_tailroom(skb),
	624	skb_headlen(skb),
	625	sp->remain);
	626
	627	skb->ip_summed = CHECKSUM_UNNECESSARY;
	628	}
	629
	630	_debug("append");
	631	sp = rxrpc_skb(skb);
	632
	633	/* append next segment of data to the current buffer */
	634	copy = skb_tailroom(skb);
	635	ASSERTCMP(copy, >, 0);
	636	if (copy > segment)
	637	copy = segment;
	638	if (copy > sp->remain)
	639	copy = sp->remain;
	640
	641	_debug("add");
	642	ret = skb_add_data(skb, from, copy);
	643	_debug("added");
	644	if (ret < 0)
	645	goto efault;
	646	sp->remain -= copy;
	647	skb->mark += copy;
19e6454c	648	copied += copy;
17926a79 DH	649
	650	len -= copy;
	651	segment -= copy;
	652	from += copy;
	653	while (segment == 0 && ioc > 0) {
	654	from = iov->iov_base;
	655	segment = iov->iov_len;
	656	iov++;
	657	ioc--;
	658	}
	659	if (len == 0) {
	660	segment = 0;
	661	ioc = 0;
	662	}
	663
	664	/* check for the far side aborting the call or a network error
	665	* occurring */
	666	if (call->state > RXRPC_CALL_COMPLETE)
	667	goto call_aborted;
	668
	669	/* add the packet to the send queue if it's now full */
	670	if (sp->remain <= 0 \|\| (segment == 0 && !more)) {
	671	struct rxrpc_connection *conn = call->conn;
e8388eb1	672	uint32_t seq;
17926a79 DH	673	size_t pad;
	674
	675	/* pad out if we're using security */
	676	if (conn->security) {
	677	pad = conn->security_size + skb->mark;
	678	pad = conn->size_align - pad;
	679	pad &= conn->size_align - 1;
	680	_debug("pad %zu", pad);
	681	if (pad)
	682	memset(skb_put(skb, pad), 0, pad);
	683	}
	684
e8388eb1 DH	685	seq = atomic_inc_return(&call->sequence);
e8388eb1 DH	686
17926a79 DH	687	sp->hdr.epoch = conn->epoch;
	688	sp->hdr.cid = call->cid;
	689	sp->hdr.callNumber = call->call_id;
e8388eb1	690	sp->hdr.seq = htonl(seq);
17926a79 DH	691	sp->hdr.serial =
	692	htonl(atomic_inc_return(&conn->serial));
	693	sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
	694	sp->hdr.userStatus = 0;
	695	sp->hdr.securityIndex = conn->security_ix;
	696	sp->hdr._rsvd = 0;
	697	sp->hdr.serviceId = conn->service_id;
	698
	699	sp->hdr.flags = conn->out_clientflag;
	700	if (len == 0 && !more)
	701	sp->hdr.flags \|= RXRPC_LAST_PACKET;
	702	else if (CIRC_SPACE(call->acks_head, call->acks_tail,
	703	call->acks_winsz) > 1)
	704	sp->hdr.flags \|= RXRPC_MORE_PACKETS;
e8388eb1 DH	705	if (more && seq & 1)
e8388eb1 DH	706	sp->hdr.flags \|= RXRPC_REQUEST_ACK;
17926a79 DH	707
	708	ret = rxrpc_secure_packet(
	709	call, skb, skb->mark,
	710	skb->head + sizeof(struct rxrpc_header));
	711	if (ret < 0)
	712	goto out;
	713
	714	memcpy(skb->head, &sp->hdr,
	715	sizeof(struct rxrpc_header));
	716	rxrpc_queue_packet(call, skb, segment == 0 && !more);
	717	skb = NULL;
	718	}
	719
	720	} while (segment > 0);
	721
19e6454c DH	722	success:
19e6454c DH	723	ret = copied;
17926a79 DH	724	out:
	725	call->tx_pending = skb;
	726	_leave(" = %d", ret);
	727	return ret;
	728
	729	call_aborted:
	730	rxrpc_free_skb(skb);
	731	if (call->state == RXRPC_CALL_NETWORK_ERROR)
	732	ret = call->conn->trans->peer->net_error;
	733	else
	734	ret = -ECONNABORTED;
	735	_leave(" = %d", ret);
	736	return ret;
	737
	738	maybe_error:
	739	if (copied)
19e6454c	740	goto success;
17926a79 DH	741	goto out;
	742
	743	efault:
	744	ret = -EFAULT;
	745	goto out;
	746	}