[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_dp_mst.c

/*
 * Copyright © 2008 Intel Corporation
 *             2014 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include <drm/drmP.h>
#include "i915_drv.h"
#include "intel_drv.h"
#include <drm/drm_atomic_helper.h>
#include <drm/drm_crtc_helper.h>
#include <drm/drm_edid.h>

static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
					struct intel_crtc_state *pipe_config)
{
	struct drm_device *dev = encoder->base.dev;
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_dp *intel_dp = &intel_dig_port->dp;
	struct drm_atomic_state *state;
	int bpp, i;
	int lane_count, slots;
	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
	struct drm_connector *drm_connector;
	struct intel_connector *connector, *found = NULL;
	struct drm_connector_state *connector_state;
	int mst_pbn;

	pipe_config->dp_encoder_is_mst = true;
	pipe_config->has_pch_encoder = false;
	pipe_config->has_dp_encoder = true;
	bpp = 24;
	/*
	 * for MST we always configure max link bw - the spec doesn't
	 * seem to suggest we should do otherwise.
	 */
	lane_count = drm_dp_max_lane_count(intel_dp->dpcd);


	pipe_config->lane_count = lane_count;

	pipe_config->pipe_bpp = 24;
	pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);

	state = pipe_config->base.state;

	for_each_connector_in_state(state, drm_connector, connector_state, i) {
		connector = to_intel_connector(drm_connector);

		if (connector_state->best_encoder == &encoder->base) {
			found = connector;
			break;
		}
	}

	if (!found) {
		DRM_ERROR("can't find connector\n");
		return false;
	}

	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);

	pipe_config->pbn = mst_pbn;
	slots = drm_dp_find_vcpi_slots(&intel_dp->mst_mgr, mst_pbn);

	intel_link_compute_m_n(bpp, lane_count,
			       adjusted_mode->crtc_clock,
			       pipe_config->port_clock,
			       &pipe_config->dp_m_n);

	pipe_config->dp_m_n.tu = slots;

	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
		hsw_dp_set_ddi_pll_sel(pipe_config);

	return true;

}

static void intel_mst_disable_dp(struct intel_encoder *encoder)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_dp *intel_dp = &intel_dig_port->dp;
	int ret;

	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);

	drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, intel_mst->port);

	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
	if (ret) {
		DRM_ERROR("failed to update payload %d\n", ret);
	}
}

static void intel_mst_post_disable_dp(struct intel_encoder *encoder)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_dp *intel_dp = &intel_dig_port->dp;

	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);

	/* this can fail */
	drm_dp_check_act_status(&intel_dp->mst_mgr);
	/* and this can also fail */
	drm_dp_update_payload_part2(&intel_dp->mst_mgr);

	drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, intel_mst->port);

	intel_dp->active_mst_links--;
	intel_mst->port = NULL;
	if (intel_dp->active_mst_links == 0) {
		intel_dig_port->base.post_disable(&intel_dig_port->base);
		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
	}
}

static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_dp *intel_dp = &intel_dig_port->dp;
	struct drm_device *dev = encoder->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum port port = intel_dig_port->port;
	int ret;
	uint32_t temp;
	struct intel_connector *found = NULL, *connector;
	int slots;
	struct drm_crtc *crtc = encoder->base.crtc;
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);

	for_each_intel_connector(dev, connector) {
		if (connector->base.state->best_encoder == &encoder->base) {
			found = connector;
			break;
		}
	}

	if (!found) {
		DRM_ERROR("can't find connector\n");
		return;
	}

	/* MST encoders are bound to a crtc, not to a connector,
	 * force the mapping here for get_hw_state.
	 */
	found->encoder = encoder;

	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
	intel_mst->port = found->port;

	if (intel_dp->active_mst_links == 0) {
		intel_ddi_clk_select(encoder, intel_crtc->config);

		intel_dp_set_link_params(intel_dp, intel_crtc->config);

		intel_ddi_init_dp_buf_reg(&intel_dig_port->base);

		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);

		intel_dp_start_link_train(intel_dp);
		intel_dp_stop_link_train(intel_dp);
	}

	ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr,
				       intel_mst->port,
				       intel_crtc->config->pbn, &slots);
	if (ret == false) {
		DRM_ERROR("failed to allocate vcpi\n");
		return;
	}


	intel_dp->active_mst_links++;
	temp = I915_READ(DP_TP_STATUS(port));
	I915_WRITE(DP_TP_STATUS(port), temp);

	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
}

static void intel_mst_enable_dp(struct intel_encoder *encoder)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_dp *intel_dp = &intel_dig_port->dp;
	struct drm_device *dev = intel_dig_port->base.base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum port port = intel_dig_port->port;
	int ret;

	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);

	if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_ACT_SENT),
		     1))
		DRM_ERROR("Timed out waiting for ACT sent\n");

	ret = drm_dp_check_act_status(&intel_dp->mst_mgr);

	ret = drm_dp_update_payload_part2(&intel_dp->mst_mgr);
}

static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder,
				      enum pipe *pipe)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	*pipe = intel_mst->pipe;
	if (intel_mst->port)
		return true;
	return false;
}

static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
					struct intel_crtc_state *pipe_config)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
	struct drm_device *dev = encoder->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
	u32 temp, flags = 0;

	pipe_config->has_dp_encoder = true;

	temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
	if (temp & TRANS_DDI_PHSYNC)
		flags |= DRM_MODE_FLAG_PHSYNC;
	else
		flags |= DRM_MODE_FLAG_NHSYNC;
	if (temp & TRANS_DDI_PVSYNC)
		flags |= DRM_MODE_FLAG_PVSYNC;
	else
		flags |= DRM_MODE_FLAG_NVSYNC;

	switch (temp & TRANS_DDI_BPC_MASK) {
	case TRANS_DDI_BPC_6:
		pipe_config->pipe_bpp = 18;
		break;
	case TRANS_DDI_BPC_8:
		pipe_config->pipe_bpp = 24;
		break;
	case TRANS_DDI_BPC_10:
		pipe_config->pipe_bpp = 30;
		break;
	case TRANS_DDI_BPC_12:
		pipe_config->pipe_bpp = 36;
		break;
	default:
		break;
	}
	pipe_config->base.adjusted_mode.flags |= flags;

	pipe_config->lane_count =
		((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;

	intel_dp_get_m_n(crtc, pipe_config);

	intel_ddi_clock_get(&intel_dig_port->base, pipe_config);
}

static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct intel_dp *intel_dp = intel_connector->mst_port;
	struct edid *edid;
	int ret;

	edid = drm_dp_mst_get_edid(connector, &intel_dp->mst_mgr, intel_connector->port);
	if (!edid)
		return 0;

	ret = intel_connector_update_modes(connector, edid);
	kfree(edid);

	return ret;
}

static enum drm_connector_status
intel_dp_mst_detect(struct drm_connector *connector, bool force)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct intel_dp *intel_dp = intel_connector->mst_port;

	return drm_dp_mst_detect_port(connector, &intel_dp->mst_mgr, intel_connector->port);
}

static int
intel_dp_mst_set_property(struct drm_connector *connector,
			  struct drm_property *property,
			  uint64_t val)
{
	return 0;
}

static void
intel_dp_mst_connector_destroy(struct drm_connector *connector)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);

	if (!IS_ERR_OR_NULL(intel_connector->edid))
		kfree(intel_connector->edid);

	drm_connector_cleanup(connector);
	kfree(connector);
}

static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
	.dpms = drm_atomic_helper_connector_dpms,
	.detect = intel_dp_mst_detect,
	.fill_modes = drm_helper_probe_single_connector_modes,
	.set_property = intel_dp_mst_set_property,
	.atomic_get_property = intel_connector_atomic_get_property,
	.destroy = intel_dp_mst_connector_destroy,
	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
};

static int intel_dp_mst_get_modes(struct drm_connector *connector)
{
	return intel_dp_mst_get_ddc_modes(connector);
}

static enum drm_mode_status
intel_dp_mst_mode_valid(struct drm_connector *connector,
			struct drm_display_mode *mode)
{
	/* TODO - validate mode against available PBN for link */
	if (mode->clock < 10000)
		return MODE_CLOCK_LOW;

	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
		return MODE_H_ILLEGAL;

	return MODE_OK;
}

static struct drm_encoder *intel_mst_atomic_best_encoder(struct drm_connector *connector,
							 struct drm_connector_state *state)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct intel_dp *intel_dp = intel_connector->mst_port;
	struct intel_crtc *crtc = to_intel_crtc(state->crtc);

	return &intel_dp->mst_encoders[crtc->pipe]->base.base;
}

static struct drm_encoder *intel_mst_best_encoder(struct drm_connector *connector)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct intel_dp *intel_dp = intel_connector->mst_port;
	return &intel_dp->mst_encoders[0]->base.base;
}

static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = {
	.get_modes = intel_dp_mst_get_modes,
	.mode_valid = intel_dp_mst_mode_valid,
	.atomic_best_encoder = intel_mst_atomic_best_encoder,
	.best_encoder = intel_mst_best_encoder,
};

static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder)
{
	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);

	drm_encoder_cleanup(encoder);
	kfree(intel_mst);
}

static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = {
	.destroy = intel_dp_mst_encoder_destroy,
};

static bool intel_dp_mst_get_hw_state(struct intel_connector *connector)
{
	if (connector->encoder && connector->base.state->crtc) {
		enum pipe pipe;
		if (!connector->encoder->get_hw_state(connector->encoder, &pipe))
			return false;
		return true;
	}
	return false;
}

static void intel_connector_add_to_fbdev(struct intel_connector *connector)
{
#ifdef CONFIG_DRM_FBDEV_EMULATION
	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);

	if (dev_priv->fbdev)
		drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper,
						&connector->base);
#endif
}

static void intel_connector_remove_from_fbdev(struct intel_connector *connector)
{
#ifdef CONFIG_DRM_FBDEV_EMULATION
	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);

	if (dev_priv->fbdev)
		drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper,
						   &connector->base);
#endif
}

static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port, const char *pathprop)
{
	struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr);
	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
	struct drm_device *dev = intel_dig_port->base.base.dev;
	struct intel_connector *intel_connector;
	struct drm_connector *connector;
	int i;

	intel_connector = intel_connector_alloc();
	if (!intel_connector)
		return NULL;

	connector = &intel_connector->base;
	drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort);
	drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs);

	intel_connector->unregister = intel_connector_unregister;
	intel_connector->get_hw_state = intel_dp_mst_get_hw_state;
	intel_connector->mst_port = intel_dp;
	intel_connector->port = port;

	for (i = PIPE_A; i <= PIPE_C; i++) {
		drm_mode_connector_attach_encoder(&intel_connector->base,
						  &intel_dp->mst_encoders[i]->base.base);
	}
	intel_dp_add_properties(intel_dp, connector);

	drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0);
	drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0);

	drm_mode_connector_set_path_property(connector, pathprop);
	return connector;
}

static void intel_dp_register_mst_connector(struct drm_connector *connector)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct drm_device *dev = connector->dev;
	drm_modeset_lock_all(dev);
	intel_connector_add_to_fbdev(intel_connector);
	drm_modeset_unlock_all(dev);
	drm_connector_register(&intel_connector->base);
}

static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
					   struct drm_connector *connector)
{
	struct intel_connector *intel_connector = to_intel_connector(connector);
	struct drm_device *dev = connector->dev;

	/* need to nuke the connector */
	drm_modeset_lock_all(dev);
	if (connector->state->crtc) {
		struct drm_mode_set set;
		int ret;

		memset(&set, 0, sizeof(set));
		set.crtc = connector->state->crtc,

		ret = drm_atomic_helper_set_config(&set);

		WARN(ret, "Disabling mst crtc failed with %i\n", ret);
	}
	drm_modeset_unlock_all(dev);

	intel_connector->unregister(intel_connector);

	drm_modeset_lock_all(dev);
	intel_connector_remove_from_fbdev(intel_connector);
	drm_connector_cleanup(connector);
	drm_modeset_unlock_all(dev);

	kfree(intel_connector);
	DRM_DEBUG_KMS("\n");
}

static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
{
	struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr);
	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
	struct drm_device *dev = intel_dig_port->base.base.dev;

	drm_kms_helper_hotplug_event(dev);
}

static struct drm_dp_mst_topology_cbs mst_cbs = {
	.add_connector = intel_dp_add_mst_connector,
	.register_connector = intel_dp_register_mst_connector,
	.destroy_connector = intel_dp_destroy_mst_connector,
	.hotplug = intel_dp_mst_hotplug,
};

static struct intel_dp_mst_encoder *
intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum pipe pipe)
{
	struct intel_dp_mst_encoder *intel_mst;
	struct intel_encoder *intel_encoder;
	struct drm_device *dev = intel_dig_port->base.base.dev;

	intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL);

	if (!intel_mst)
		return NULL;

	intel_mst->pipe = pipe;
	intel_encoder = &intel_mst->base;
	intel_mst->primary = intel_dig_port;

	drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs,
			 DRM_MODE_ENCODER_DPMST, NULL);

	intel_encoder->type = INTEL_OUTPUT_DP_MST;
	intel_encoder->crtc_mask = 0x7;
	intel_encoder->cloneable = 0;

	intel_encoder->compute_config = intel_dp_mst_compute_config;
	intel_encoder->disable = intel_mst_disable_dp;
	intel_encoder->post_disable = intel_mst_post_disable_dp;
	intel_encoder->pre_enable = intel_mst_pre_enable_dp;
	intel_encoder->enable = intel_mst_enable_dp;
	intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state;
	intel_encoder->get_config = intel_dp_mst_enc_get_config;

	return intel_mst;

}

static bool
intel_dp_create_fake_mst_encoders(struct intel_digital_port *intel_dig_port)
{
	int i;
	struct intel_dp *intel_dp = &intel_dig_port->dp;

	for (i = PIPE_A; i <= PIPE_C; i++)
		intel_dp->mst_encoders[i] = intel_dp_create_fake_mst_encoder(intel_dig_port, i);
	return true;
}

int
intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_base_id)
{
	struct intel_dp *intel_dp = &intel_dig_port->dp;
	struct drm_device *dev = intel_dig_port->base.base.dev;
	int ret;

	intel_dp->can_mst = true;
	intel_dp->mst_mgr.cbs = &mst_cbs;

	/* create encoders */
	intel_dp_create_fake_mst_encoders(intel_dig_port);
	ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, dev->dev, &intel_dp->aux, 16, 3, conn_base_id);
	if (ret) {
		intel_dp->can_mst = false;
		return ret;
	}
	return 0;
}

void
intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port)
{
	struct intel_dp *intel_dp = &intel_dig_port->dp;

	if (!intel_dp->can_mst)
		return;

	drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr);
	/* encoders will get killed by normal cleanup */
}
Commit	Line	Data
0e32b39c DA	1	/*
	2	* Copyright © 2008 Intel Corporation
	3	* 2014 Red Hat Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice (including the next
	13	* paragraph) shall be included in all copies or substantial portions of the
	14	* Software.
	15	*
	16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	21	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	22	* IN THE SOFTWARE.
	23	*
	24	*/
	25
	26	#include <drm/drmP.h>
	27	#include "i915_drv.h"
	28	#include "intel_drv.h"
c6f95f27	29	#include <drm/drm_atomic_helper.h>
0e32b39c DA	30	#include <drm/drm_crtc_helper.h>
	31	#include <drm/drm_edid.h>
	32
	33	static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
5cec258b	34	struct intel_crtc_state *pipe_config)
0e32b39c	35	{
6fa2d197	36	struct drm_device *dev = encoder->base.dev;
0e32b39c DA	37	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	38	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	39	struct intel_dp *intel_dp = &intel_dig_port->dp;
e75f4771 ACO	40	struct drm_atomic_state *state;
e75f4771 ACO	41	int bpp, i;
04a60f9f	42	int lane_count, slots;
7c5f93b0	43	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
da3ced29 ACO	44	struct drm_connector *drm_connector;
	45	struct intel_connector connector, found = NULL;
	46	struct drm_connector_state *connector_state;
0e32b39c DA	47	int mst_pbn;
	48
	49	pipe_config->dp_encoder_is_mst = true;
	50	pipe_config->has_pch_encoder = false;
	51	pipe_config->has_dp_encoder = true;
	52	bpp = 24;
	53	/*
	54	* for MST we always configure max link bw - the spec doesn't
	55	* seem to suggest we should do otherwise.
	56	*/
	57	lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
ed4e9c1d	58
ed4e9c1d	59
90a6b7b0	60	pipe_config->lane_count = lane_count;
0e32b39c DA	61
0e32b39c DA	62	pipe_config->pipe_bpp = 24;
04a60f9f	63	pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);
0e32b39c	64
e75f4771 ACO	65	state = pipe_config->base.state;
e75f4771 ACO	66
da3ced29 ACO	67	for_each_connector_in_state(state, drm_connector, connector_state, i) {
da3ced29 ACO	68	connector = to_intel_connector(drm_connector);
e75f4771	69
da3ced29 ACO	70	if (connector_state->best_encoder == &encoder->base) {
da3ced29 ACO	71	found = connector;
0e32b39c DA	72	break;
	73	}
	74	}
	75
	76	if (!found) {
	77	DRM_ERROR("can't find connector\n");
	78	return false;
	79	}
	80
aad941d5	81	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
0e32b39c DA	82
	83	pipe_config->pbn = mst_pbn;
	84	slots = drm_dp_find_vcpi_slots(&intel_dp->mst_mgr, mst_pbn);
	85
	86	intel_link_compute_m_n(bpp, lane_count,
	87	adjusted_mode->crtc_clock,
	88	pipe_config->port_clock,
	89	&pipe_config->dp_m_n);
	90
	91	pipe_config->dp_m_n.tu = slots;
6fa2d197 ACO	92
	93	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
	94	hsw_dp_set_ddi_pll_sel(pipe_config);
	95
0e32b39c DA	96	return true;
	97
	98	}
	99
	100	static void intel_mst_disable_dp(struct intel_encoder *encoder)
	101	{
	102	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	103	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	104	struct intel_dp *intel_dp = &intel_dig_port->dp;
	105	int ret;
	106
	107	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
	108
	109	drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, intel_mst->port);
	110
	111	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
	112	if (ret) {
	113	DRM_ERROR("failed to update payload %d\n", ret);
	114	}
	115	}
	116
	117	static void intel_mst_post_disable_dp(struct intel_encoder *encoder)
	118	{
	119	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	120	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	121	struct intel_dp *intel_dp = &intel_dig_port->dp;
	122
	123	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
	124
	125	/* this can fail */
	126	drm_dp_check_act_status(&intel_dp->mst_mgr);
	127	/* and this can also fail */
	128	drm_dp_update_payload_part2(&intel_dp->mst_mgr);
	129
	130	drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, intel_mst->port);
	131
	132	intel_dp->active_mst_links--;
	133	intel_mst->port = NULL;
	134	if (intel_dp->active_mst_links == 0) {
	135	intel_dig_port->base.post_disable(&intel_dig_port->base);
	136	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
	137	}
	138	}
	139
	140	static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
	141	{
	142	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	143	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	144	struct intel_dp *intel_dp = &intel_dig_port->dp;
	145	struct drm_device *dev = encoder->base.dev;
	146	struct drm_i915_private *dev_priv = dev->dev_private;
	147	enum port port = intel_dig_port->port;
	148	int ret;
	149	uint32_t temp;
9b4fd8f2	150	struct intel_connector found = NULL, connector;
0e32b39c DA	151	int slots;
	152	struct drm_crtc *crtc = encoder->base.crtc;
	153	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
	154
9b4fd8f2 ACO	155	for_each_intel_connector(dev, connector) {
	156	if (connector->base.state->best_encoder == &encoder->base) {
	157	found = connector;
0e32b39c DA	158	break;
	159	}
	160	}
	161
	162	if (!found) {
	163	DRM_ERROR("can't find connector\n");
	164	return;
	165	}
	166
e85376cb ML	167	/* MST encoders are bound to a crtc, not to a connector,
	168	* force the mapping here for get_hw_state.
	169	*/
	170	found->encoder = encoder;
	171
0e32b39c DA	172	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
	173	intel_mst->port = found->port;
	174
	175	if (intel_dp->active_mst_links == 0) {
e404ba8d	176	intel_ddi_clk_select(encoder, intel_crtc->config);
0e32b39c	177
901c2daf VS	178	intel_dp_set_link_params(intel_dp, intel_crtc->config);
901c2daf VS	179
0e32b39c DA	180	intel_ddi_init_dp_buf_reg(&intel_dig_port->base);
	181
	182	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
	183
0e32b39c	184	intel_dp_start_link_train(intel_dp);
0e32b39c DA	185	intel_dp_stop_link_train(intel_dp);
	186	}
	187
	188	ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr,
6e3c9717 ACO	189	intel_mst->port,
6e3c9717 ACO	190	intel_crtc->config->pbn, &slots);
0e32b39c DA	191	if (ret == false) {
	192	DRM_ERROR("failed to allocate vcpi\n");
	193	return;
	194	}
	195
	196
	197	intel_dp->active_mst_links++;
	198	temp = I915_READ(DP_TP_STATUS(port));
	199	I915_WRITE(DP_TP_STATUS(port), temp);
	200
	201	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
	202	}
	203
	204	static void intel_mst_enable_dp(struct intel_encoder *encoder)
	205	{
	206	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	207	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	208	struct intel_dp *intel_dp = &intel_dig_port->dp;
	209	struct drm_device *dev = intel_dig_port->base.base.dev;
	210	struct drm_i915_private *dev_priv = dev->dev_private;
	211	enum port port = intel_dig_port->port;
	212	int ret;
	213
	214	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
	215
	216	if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_ACT_SENT),
	217	1))
	218	DRM_ERROR("Timed out waiting for ACT sent\n");
	219
	220	ret = drm_dp_check_act_status(&intel_dp->mst_mgr);
	221
	222	ret = drm_dp_update_payload_part2(&intel_dp->mst_mgr);
	223	}
	224
	225	static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder,
	226	enum pipe *pipe)
	227	{
	228	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	229	*pipe = intel_mst->pipe;
	230	if (intel_mst->port)
	231	return true;
	232	return false;
	233	}
	234
	235	static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
5cec258b	236	struct intel_crtc_state *pipe_config)
0e32b39c DA	237	{
	238	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
	239	struct intel_digital_port *intel_dig_port = intel_mst->primary;
	240	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
	241	struct drm_device *dev = encoder->base.dev;
	242	struct drm_i915_private *dev_priv = dev->dev_private;
0cb09a97	243	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
0e32b39c DA	244	u32 temp, flags = 0;
	245
	246	pipe_config->has_dp_encoder = true;
	247
	248	temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
	249	if (temp & TRANS_DDI_PHSYNC)
	250	flags \|= DRM_MODE_FLAG_PHSYNC;
	251	else
	252	flags \|= DRM_MODE_FLAG_NHSYNC;
	253	if (temp & TRANS_DDI_PVSYNC)
	254	flags \|= DRM_MODE_FLAG_PVSYNC;
	255	else
	256	flags \|= DRM_MODE_FLAG_NVSYNC;
	257
	258	switch (temp & TRANS_DDI_BPC_MASK) {
	259	case TRANS_DDI_BPC_6:
	260	pipe_config->pipe_bpp = 18;
	261	break;
	262	case TRANS_DDI_BPC_8:
	263	pipe_config->pipe_bpp = 24;
	264	break;
	265	case TRANS_DDI_BPC_10:
	266	pipe_config->pipe_bpp = 30;
	267	break;
	268	case TRANS_DDI_BPC_12:
	269	pipe_config->pipe_bpp = 36;
	270	break;
	271	default:
	272	break;
	273	}
2d112de7	274	pipe_config->base.adjusted_mode.flags \|= flags;
90a6b7b0 VS	275
	276	pipe_config->lane_count =
	277	((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
	278
0e32b39c DA	279	intel_dp_get_m_n(crtc, pipe_config);
	280
	281	intel_ddi_clock_get(&intel_dig_port->base, pipe_config);
	282	}
	283
	284	static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector)
	285	{
	286	struct intel_connector *intel_connector = to_intel_connector(connector);
	287	struct intel_dp *intel_dp = intel_connector->mst_port;
	288	struct edid *edid;
	289	int ret;
	290
	291	edid = drm_dp_mst_get_edid(connector, &intel_dp->mst_mgr, intel_connector->port);
	292	if (!edid)
	293	return 0;
	294
	295	ret = intel_connector_update_modes(connector, edid);
	296	kfree(edid);
	297
	298	return ret;
	299	}
	300
	301	static enum drm_connector_status
f7f3d48a	302	intel_dp_mst_detect(struct drm_connector *connector, bool force)
0e32b39c DA	303	{
	304	struct intel_connector *intel_connector = to_intel_connector(connector);
	305	struct intel_dp *intel_dp = intel_connector->mst_port;
	306
c6a0aed4	307	return drm_dp_mst_detect_port(connector, &intel_dp->mst_mgr, intel_connector->port);
0e32b39c DA	308	}
0e32b39c DA	309
0e32b39c DA	310	static int
	311	intel_dp_mst_set_property(struct drm_connector *connector,
	312	struct drm_property *property,
	313	uint64_t val)
	314	{
	315	return 0;
	316	}
	317
	318	static void
	319	intel_dp_mst_connector_destroy(struct drm_connector *connector)
	320	{
	321	struct intel_connector *intel_connector = to_intel_connector(connector);
	322
	323	if (!IS_ERR_OR_NULL(intel_connector->edid))
	324	kfree(intel_connector->edid);
	325
	326	drm_connector_cleanup(connector);
	327	kfree(connector);
	328	}
	329
	330	static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
4d688a2a	331	.dpms = drm_atomic_helper_connector_dpms,
0e32b39c DA	332	.detect = intel_dp_mst_detect,
	333	.fill_modes = drm_helper_probe_single_connector_modes,
	334	.set_property = intel_dp_mst_set_property,
2545e4a6	335	.atomic_get_property = intel_connector_atomic_get_property,
0e32b39c	336	.destroy = intel_dp_mst_connector_destroy,
c6f95f27	337	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
98969725	338	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
0e32b39c DA	339	};
	340
	341	static int intel_dp_mst_get_modes(struct drm_connector *connector)
	342	{
	343	return intel_dp_mst_get_ddc_modes(connector);
	344	}
	345
	346	static enum drm_mode_status
	347	intel_dp_mst_mode_valid(struct drm_connector *connector,
	348	struct drm_display_mode *mode)
	349	{
	350	/* TODO - validate mode against available PBN for link */
	351	if (mode->clock < 10000)
	352	return MODE_CLOCK_LOW;
	353
	354	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
	355	return MODE_H_ILLEGAL;
	356
	357	return MODE_OK;
	358	}
	359
459485ad DV	360	static struct drm_encoder intel_mst_atomic_best_encoder(struct drm_connector connector,
	361	struct drm_connector_state *state)
	362	{
	363	struct intel_connector *intel_connector = to_intel_connector(connector);
	364	struct intel_dp *intel_dp = intel_connector->mst_port;
	365	struct intel_crtc *crtc = to_intel_crtc(state->crtc);
	366
	367	return &intel_dp->mst_encoders[crtc->pipe]->base.base;
	368	}
	369
0e32b39c DA	370	static struct drm_encoder intel_mst_best_encoder(struct drm_connector connector)
	371	{
	372	struct intel_connector *intel_connector = to_intel_connector(connector);
	373	struct intel_dp *intel_dp = intel_connector->mst_port;
	374	return &intel_dp->mst_encoders[0]->base.base;
	375	}
	376
	377	static const struct drm_connector_helper_funcs intel_dp_mst_connector_helper_funcs = {
	378	.get_modes = intel_dp_mst_get_modes,
	379	.mode_valid = intel_dp_mst_mode_valid,
459485ad	380	.atomic_best_encoder = intel_mst_atomic_best_encoder,
0e32b39c DA	381	.best_encoder = intel_mst_best_encoder,
	382	};
	383
	384	static void intel_dp_mst_encoder_destroy(struct drm_encoder *encoder)
	385	{
	386	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
	387
	388	drm_encoder_cleanup(encoder);
	389	kfree(intel_mst);
	390	}
	391
	392	static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = {
	393	.destroy = intel_dp_mst_encoder_destroy,
	394	};
	395
	396	static bool intel_dp_mst_get_hw_state(struct intel_connector *connector)
	397	{
e85376cb	398	if (connector->encoder && connector->base.state->crtc) {
0e32b39c DA	399	enum pipe pipe;
	400	if (!connector->encoder->get_hw_state(connector->encoder, &pipe))
	401	return false;
	402	return true;
	403	}
	404	return false;
	405	}
	406
7296c849 CW	407	static void intel_connector_add_to_fbdev(struct intel_connector *connector)
7296c849 CW	408	{
0695726e	409	#ifdef CONFIG_DRM_FBDEV_EMULATION
7296c849	410	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
54632abe LW	411
	412	if (dev_priv->fbdev)
	413	drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper,
	414	&connector->base);
7296c849 CW	415	#endif
	416	}
	417
	418	static void intel_connector_remove_from_fbdev(struct intel_connector *connector)
	419	{
0695726e	420	#ifdef CONFIG_DRM_FBDEV_EMULATION
7296c849	421	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
54632abe LW	422
	423	if (dev_priv->fbdev)
	424	drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper,
	425	&connector->base);
7296c849 CW	426	#endif
	427	}
	428
12e6cecd	429	static struct drm_connector intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr mgr, struct drm_dp_mst_port port, const char pathprop)
0e32b39c DA	430	{
	431	struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr);
	432	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
	433	struct drm_device *dev = intel_dig_port->base.base.dev;
0e32b39c DA	434	struct intel_connector *intel_connector;
	435	struct drm_connector *connector;
	436	int i;
	437
9bdbd0b9	438	intel_connector = intel_connector_alloc();
0e32b39c DA	439	if (!intel_connector)
	440	return NULL;
	441
	442	connector = &intel_connector->base;
	443	drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort);
	444	drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs);
	445
	446	intel_connector->unregister = intel_connector_unregister;
	447	intel_connector->get_hw_state = intel_dp_mst_get_hw_state;
	448	intel_connector->mst_port = intel_dp;
	449	intel_connector->port = port;
	450
	451	for (i = PIPE_A; i <= PIPE_C; i++) {
	452	drm_mode_connector_attach_encoder(&intel_connector->base,
	453	&intel_dp->mst_encoders[i]->base.base);
	454	}
	455	intel_dp_add_properties(intel_dp, connector);
	456
	457	drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0);
6f134d7b DA	458	drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0);
6f134d7b DA	459
0e32b39c	460	drm_mode_connector_set_path_property(connector, pathprop);
d9515c5e DA	461	return connector;
	462	}
	463
	464	static void intel_dp_register_mst_connector(struct drm_connector *connector)
	465	{
	466	struct intel_connector *intel_connector = to_intel_connector(connector);
	467	struct drm_device *dev = connector->dev;
8bb4da1d	468	drm_modeset_lock_all(dev);
7296c849	469	intel_connector_add_to_fbdev(intel_connector);
8bb4da1d	470	drm_modeset_unlock_all(dev);
0e32b39c	471	drm_connector_register(&intel_connector->base);
0e32b39c DA	472	}
	473
	474	static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
	475	struct drm_connector *connector)
	476	{
	477	struct intel_connector *intel_connector = to_intel_connector(connector);
	478	struct drm_device *dev = connector->dev;
20fae983	479
0e32b39c	480	/* need to nuke the connector */
8bb4da1d	481	drm_modeset_lock_all(dev);
20fae983 ML	482	if (connector->state->crtc) {
	483	struct drm_mode_set set;
	484	int ret;
	485
	486	memset(&set, 0, sizeof(set));
	487	set.crtc = connector->state->crtc,
	488
	489	ret = drm_atomic_helper_set_config(&set);
	490
	491	WARN(ret, "Disabling mst crtc failed with %i\n", ret);
	492	}
8bb4da1d	493	drm_modeset_unlock_all(dev);
0e32b39c DA	494
	495	intel_connector->unregister(intel_connector);
	496
8bb4da1d	497	drm_modeset_lock_all(dev);
7296c849	498	intel_connector_remove_from_fbdev(intel_connector);
0e32b39c	499	drm_connector_cleanup(connector);
8bb4da1d	500	drm_modeset_unlock_all(dev);
0e32b39c	501
0e32b39c DA	502	kfree(intel_connector);
	503	DRM_DEBUG_KMS("\n");
	504	}
	505
	506	static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
	507	{
	508	struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr);
	509	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
	510	struct drm_device *dev = intel_dig_port->base.base.dev;
	511
	512	drm_kms_helper_hotplug_event(dev);
	513	}
	514
	515	static struct drm_dp_mst_topology_cbs mst_cbs = {
	516	.add_connector = intel_dp_add_mst_connector,
d9515c5e	517	.register_connector = intel_dp_register_mst_connector,
0e32b39c DA	518	.destroy_connector = intel_dp_destroy_mst_connector,
	519	.hotplug = intel_dp_mst_hotplug,
	520	};
	521
	522	static struct intel_dp_mst_encoder *
	523	intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum pipe pipe)
	524	{
	525	struct intel_dp_mst_encoder *intel_mst;
	526	struct intel_encoder *intel_encoder;
	527	struct drm_device *dev = intel_dig_port->base.base.dev;
	528
	529	intel_mst = kzalloc(sizeof(*intel_mst), GFP_KERNEL);
	530
	531	if (!intel_mst)
	532	return NULL;
	533
	534	intel_mst->pipe = pipe;
	535	intel_encoder = &intel_mst->base;
	536	intel_mst->primary = intel_dig_port;
	537
	538	drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs,
13a3d91f	539	DRM_MODE_ENCODER_DPMST, NULL);
0e32b39c DA	540
	541	intel_encoder->type = INTEL_OUTPUT_DP_MST;
	542	intel_encoder->crtc_mask = 0x7;
	543	intel_encoder->cloneable = 0;
	544
	545	intel_encoder->compute_config = intel_dp_mst_compute_config;
	546	intel_encoder->disable = intel_mst_disable_dp;
	547	intel_encoder->post_disable = intel_mst_post_disable_dp;
	548	intel_encoder->pre_enable = intel_mst_pre_enable_dp;
	549	intel_encoder->enable = intel_mst_enable_dp;
	550	intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state;
	551	intel_encoder->get_config = intel_dp_mst_enc_get_config;
	552
	553	return intel_mst;
	554
	555	}
	556
	557	static bool
	558	intel_dp_create_fake_mst_encoders(struct intel_digital_port *intel_dig_port)
	559	{
	560	int i;
	561	struct intel_dp *intel_dp = &intel_dig_port->dp;
	562
	563	for (i = PIPE_A; i <= PIPE_C; i++)
	564	intel_dp->mst_encoders[i] = intel_dp_create_fake_mst_encoder(intel_dig_port, i);
	565	return true;
	566	}
	567
	568	int
	569	intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_base_id)
	570	{
	571	struct intel_dp *intel_dp = &intel_dig_port->dp;
	572	struct drm_device *dev = intel_dig_port->base.base.dev;
	573	int ret;
	574
	575	intel_dp->can_mst = true;
	576	intel_dp->mst_mgr.cbs = &mst_cbs;
	577
	578	/* create encoders */
	579	intel_dp_create_fake_mst_encoders(intel_dig_port);
	580	ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, dev->dev, &intel_dp->aux, 16, 3, conn_base_id);
	581	if (ret) {
	582	intel_dp->can_mst = false;
	583	return ret;
	584	}
	585	return 0;
	586	}
	587
	588	void
	589	intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port)
	590	{
	591	struct intel_dp *intel_dp = &intel_dig_port->dp;
	592
	593	if (!intel_dp->can_mst)
	594	return;
	595
	596	drm_dp_mst_topology_mgr_destroy(&intel_dp->mst_mgr);
	597	/* encoders will get killed by normal cleanup */
	598	}