[deliverable/binutils-gdb.git] / gdb / nat / aarch64-sve-linux-ptrace.c

/* Common target dependent for AArch64 systems.

   Copyright (C) 2018 Free Software Foundation, Inc.

   This file is part of GDB.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#include <sys/utsname.h>
#include <sys/uio.h>
#include "common-defs.h"
#include "elf/external.h"
#include "elf/common.h"
#include "aarch64-sve-linux-ptrace.h"
#include "arch/aarch64.h"
#include "common-regcache.h"
#include "common/byte-vector.h"

static bool vq_change_warned = false;

/* See nat/aarch64-sve-linux-ptrace.h.  */

uint64_t
aarch64_sve_get_vq (int tid)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  /* Ptrace gives the vector length in bytes.  Convert it to VQ, the number of
     128bit chunks in a Z register.  We use VQ because 128bits is the minimum
     a Z register can increase in size.  */

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return 0;
    }

  uint64_t vq = sve_vq_from_vl (header.vl);

  if (!sve_vl_valid (header.vl))
    {
      warning (_("Invalid SVE state from kernel; SVE disabled."));
      return 0;
    }

  return vq;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

std::unique_ptr<gdb_byte[]>
aarch64_sve_get_sveregs (int tid)
{
  struct iovec iovec;
  struct user_sve_header header;
  uint64_t vq = aarch64_sve_get_vq (tid);

  if (vq == 0)
    perror_with_name (_("Unable to fetch SVE register header"));

  /* A ptrace call with NT_ARM_SVE will return a header followed by either a
     dump of all the SVE and FP registers, or an fpsimd structure (identical to
     the one returned by NT_FPREGSET) if the kernel has not yet executed any
     SVE code.  Make sure we allocate enough space for a full SVE dump.  */

  iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
  std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
  iovec.iov_base = buf.get ();

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    perror_with_name (_("Unable to fetch SVE registers"));

  return buf;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
				  const void *buf)
{
  char *base = (char *) buf;
  struct user_sve_header *header = (struct user_sve_header *) buf;
  uint64_t vq, vg_reg_buf = 0;

  vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);

  if (vg_reg_buf == 0)
    {
      /* VG has not been set.  */
      vg_reg_buf = sve_vg_from_vl (header->vl);
      reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
    }
  else if (vg_reg_buf != sve_vg_from_vl (header->vl) && !vq_change_warned)
    {
      /* Vector length on the running process has changed.  GDB currently does
	 not support this and will result in GDB showing incorrect partially
	 incorrect data for the vector registers.  Warn once and continue.  We
	 do not expect many programs to exhibit this behaviour.  To fix this
	 we need to spot the change earlier and generate a new target
	 descriptor.  */
      warning (_("SVE Vector length has changed (%ld to %d). "
		 "Vector registers may show incorrect data."),
	       vg_reg_buf, sve_vg_from_vl (header->vl));
      vq_change_warned = true;
    }

  if (HAS_SVE_STATE (*header))
    {
      /* The register dump contains a set of SVE registers.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
			     base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
			     base + SVE_PT_SVE_PREG_OFFSET (vq, i));

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
			   base + SVE_PT_SVE_FFR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
			   base + SVE_PT_SVE_FPSR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
			   base + SVE_PT_SVE_FPCR_OFFSET (vq));
    }
  else
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  These registers still exist in the hardware, but
	 the kernel has not yet initialised them, and so they will be null.  */

      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      /* Copy across the V registers from fpsimd structure to the Z registers,
	 ensuring the non overlapping state is set to null.  */

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	  reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	}

      reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

      /* Clear the SVE only registers.  */

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
    }
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
				    void *buf)
{
  struct user_sve_header *header = (struct user_sve_header *) buf;
  char *base = (char *) buf;
  uint64_t vq, vg_reg_buf = 0;

  vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);

  if (vg_reg_buf != 0 && vg_reg_buf != sve_vg_from_vl (header->vl))
    {
      /* Vector length on the running process has changed.  GDB currently does
	 not support this and will result in GDB writing invalid data back to
	 the vector registers.  Error and exit.  We do not expect many programs
	 to exhibit this behaviour.  To fix this we need to spot the change
	 earlier and generate a new target descriptor.  */
      error (_("SVE Vector length has changed (%ld to %d). "
	       "Cannot write back registers."),
	     vg_reg_buf, sve_vg_from_vl (header->vl));
    }

  if (!HAS_SVE_STATE (*header))
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  Where possible we want to write the reg_buf data
	 back to the kernel using the fpsimd structure.  However, if we cannot
	 then we'll need to reformat the fpsimd into a full SVE structure,
	 resulting in the initialization of SVE state written back to the
	 kernel, which is why we try to avoid it.  */

      bool has_sve_state = false;
      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      /* Check in the reg_buf if any of the Z registers are set after the
	 first 128 bits, or if any of the other SVE registers are set.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
						 zero_reg, sizeof (__int128_t));
	  if (has_sve_state)
	    break;
	}

      if (!has_sve_state)
	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	  {
	    has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
						   zero_reg, 0);
	    if (has_sve_state)
	      break;
	  }

      if (!has_sve_state)
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
						 zero_reg, 0);

      /* If no SVE state exists, then use the existing fpsimd structure to
	 write out state and return.  */
      if (!has_sve_state)
	{
	  /* The collects of the Z registers will overflow the size of a vreg.
	     There is enough space in the structure to allow for this, but we
	     cannot overflow into the next register as we might not be
	     collecting every register.  */

	  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	    {
	      if (REG_VALID
		  == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
		{
		  reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
		  memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
		}
	    }

	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

	  return;
	}

      /* Otherwise, reformat the fpsimd structure into a full SVE set, by
	 expanding the V registers (working backwards so we don't splat
	 registers before they are copied) and using null for everything else.
	 Note that enough space for a full SVE dump was originally allocated
	 for base.  */

      header->flags |= SVE_PT_REGS_SVE;
      header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);

      memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
	      sizeof (uint32_t));
      memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
	      sizeof (uint32_t));

      for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
	{
	  memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
		  sizeof (__int128_t));
	}
    }

  /* Replace the kernel values with those from reg_buf.  */

  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
			    base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

  for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
			    base + SVE_PT_SVE_PREG_OFFSET (vq, i));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
			  base + SVE_PT_SVE_FFR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
			  base + SVE_PT_SVE_FPSR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
			  base + SVE_PT_SVE_FPCR_OFFSET (vq));

}
Commit	Line	Data
122394f1 AH	1	/* Common target dependent for AArch64 systems.
	2
	3	Copyright (C) 2018 Free Software Foundation, Inc.
	4
	5	This file is part of GDB.
	6
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of the GNU General Public License as published by
	9	the Free Software Foundation; either version 3 of the License, or
	10	(at your option) any later version.
	11
	12	This program is distributed in the hope that it will be useful,
	13	but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	GNU General Public License for more details.
	16
	17	You should have received a copy of the GNU General Public License
	18	along with this program. If not, see <http://www.gnu.org/licenses/>. */
	19
	20	#include <sys/utsname.h>
	21	#include <sys/uio.h>
	22	#include "common-defs.h"
	23	#include "elf/external.h"
	24	#include "elf/common.h"
	25	#include "aarch64-sve-linux-ptrace.h"
	26	#include "arch/aarch64.h"
e9902bfc AH	27	#include "common-regcache.h"
	28	#include "common/byte-vector.h"
	29
	30	static bool vq_change_warned = false;
122394f1 AH	31
	32	/* See nat/aarch64-sve-linux-ptrace.h. */
	33
39bfb937	34	uint64_t
122394f1 AH	35	aarch64_sve_get_vq (int tid)
	36	{
	37	struct iovec iovec;
	38	struct user_sve_header header;
	39
	40	iovec.iov_len = sizeof (header);
	41	iovec.iov_base = &header;
	42
	43	/* Ptrace gives the vector length in bytes. Convert it to VQ, the number of
	44	128bit chunks in a Z register. We use VQ because 128bits is the minimum
	45	a Z register can increase in size. */
	46
	47	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	48	{
	49	/* SVE is not supported. */
	50	return 0;
	51	}
	52
e9902bfc	53	uint64_t vq = sve_vq_from_vl (header.vl);
122394f1 AH	54
	55	if (!sve_vl_valid (header.vl))
	56	{
	57	warning (_("Invalid SVE state from kernel; SVE disabled."));
	58	return 0;
	59	}
	60
	61	return vq;
	62	}
e9902bfc AH	63
	64	/* See nat/aarch64-sve-linux-ptrace.h. */
	65
	66	std::unique_ptr<gdb_byte[]>
	67	aarch64_sve_get_sveregs (int tid)
	68	{
	69	struct iovec iovec;
	70	struct user_sve_header header;
	71	uint64_t vq = aarch64_sve_get_vq (tid);
	72
	73	if (vq == 0)
	74	perror_with_name (_("Unable to fetch SVE register header"));
	75
	76	/* A ptrace call with NT_ARM_SVE will return a header followed by either a
	77	dump of all the SVE and FP registers, or an fpsimd structure (identical to
	78	the one returned by NT_FPREGSET) if the kernel has not yet executed any
	79	SVE code. Make sure we allocate enough space for a full SVE dump. */
	80
	81	iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
	82	std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
	83	iovec.iov_base = buf.get ();
	84
	85	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	86	perror_with_name (_("Unable to fetch SVE registers"));
	87
	88	return buf;
	89	}
	90
	91	/* See nat/aarch64-sve-linux-ptrace.h. */
	92
	93	void
	94	aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
	95	const void *buf)
	96	{
	97	char base = (char ) buf;
	98	struct user_sve_header header = (struct user_sve_header ) buf;
	99	uint64_t vq, vg_reg_buf = 0;
	100
	101	vq = sve_vq_from_vl (header->vl);
	102
	103	/* Sanity check the data in the header. */
	104	if (!sve_vl_valid (header->vl)
	105	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	106	error (_("Invalid SVE header from kernel."));
	107
	108	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
	109	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
	110
	111	if (vg_reg_buf == 0)
	112	{
	113	/* VG has not been set. */
	114	vg_reg_buf = sve_vg_from_vl (header->vl);
	115	reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
	116	}
	117	else if (vg_reg_buf != sve_vg_from_vl (header->vl) && !vq_change_warned)
	118	{
	119	/* Vector length on the running process has changed. GDB currently does
	120	not support this and will result in GDB showing incorrect partially
	121	incorrect data for the vector registers. Warn once and continue. We
	122	do not expect many programs to exhibit this behaviour. To fix this
	123	we need to spot the change earlier and generate a new target
	124	descriptor. */
	125	warning (_("SVE Vector length has changed (%ld to %d). "
	126	"Vector registers may show incorrect data."),
127	vg_reg_buf, sve_vg_from_vl (header->vl));
128	vq_change_warned = true;
129	}
130
131	if (HAS_SVE_STATE (*header))
132	{
133	/* The register dump contains a set of SVE registers. */
134
135	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
136	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
137	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
138
139	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
140	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
141	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
142
143	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
144	base + SVE_PT_SVE_FFR_OFFSET (vq));
145	reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
146	base + SVE_PT_SVE_FPSR_OFFSET (vq));
147	reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
148	base + SVE_PT_SVE_FPCR_OFFSET (vq));
149	}
150	else
151	{
152	/* There is no SVE state yet - the register dump contains a fpsimd
153	structure instead. These registers still exist in the hardware, but
154	the kernel has not yet initialised them, and so they will be null. */
155
156	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
157	struct user_fpsimd_state *fpsimd
158	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
159
160	/* Copy across the V registers from fpsimd structure to the Z registers,
161	ensuring the non overlapping state is set to null. */
162
163	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
164
165	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
166	{
167	memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
168	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
169	}
170
171	reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
172	reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
173
174	/* Clear the SVE only registers. */
175
176	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
177	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);
178
179	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
180	}
181	}
182
183	/* See nat/aarch64-sve-linux-ptrace.h. */
184
185	void
186	aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
187	void *buf)
188	{
189	struct user_sve_header header = (struct user_sve_header ) buf;
190	char base = (char ) buf;
191	uint64_t vq, vg_reg_buf = 0;
192
193	vq = sve_vq_from_vl (header->vl);
194
195	/* Sanity check the data in the header. */
196	if (!sve_vl_valid (header->vl)
197	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
198	error (_("Invalid SVE header from kernel."));
199
200	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
201	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
202
203	if (vg_reg_buf != 0 && vg_reg_buf != sve_vg_from_vl (header->vl))
204	{
205	/* Vector length on the running process has changed. GDB currently does
206	not support this and will result in GDB writing invalid data back to
207	the vector registers. Error and exit. We do not expect many programs
208	to exhibit this behaviour. To fix this we need to spot the change
209	earlier and generate a new target descriptor. */
210	error (_("SVE Vector length has changed (%ld to %d). "
211	"Cannot write back registers."),
212	vg_reg_buf, sve_vg_from_vl (header->vl));
213	}
214
215	if (!HAS_SVE_STATE (*header))
216	{
217	/* There is no SVE state yet - the register dump contains a fpsimd
218	structure instead. Where possible we want to write the reg_buf data
219	back to the kernel using the fpsimd structure. However, if we cannot
220	then we'll need to reformat the fpsimd into a full SVE structure,
221	resulting in the initialization of SVE state written back to the
222	kernel, which is why we try to avoid it. */
223
224	bool has_sve_state = false;
225	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
226	struct user_fpsimd_state *fpsimd
227	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
228
229	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
230
231	/* Check in the reg_buf if any of the Z registers are set after the
232	first 128 bits, or if any of the other SVE registers are set. */
233
234	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
235	{
236	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
237	zero_reg, sizeof (__int128_t));
238	if (has_sve_state)
239	break;
240	}
241
242	if (!has_sve_state)
243	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
244	{
245	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
246	zero_reg, 0);
247	if (has_sve_state)
248	break;
249	}
250
251	if (!has_sve_state)
252	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
253	zero_reg, 0);
254
255	/* If no SVE state exists, then use the existing fpsimd structure to
256	write out state and return. */
257	if (!has_sve_state)
258	{
259	/* The collects of the Z registers will overflow the size of a vreg.
260	There is enough space in the structure to allow for this, but we
261	cannot overflow into the next register as we might not be
262	collecting every register. */
263
264	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
265	{
266	if (REG_VALID
267	== reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
268	{
269	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
270	memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
271	}
272	}
273
274	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
275	reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
276	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
277	reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
278
279	return;
280	}
281
282	/* Otherwise, reformat the fpsimd structure into a full SVE set, by
283	expanding the V registers (working backwards so we don't splat
284	registers before they are copied) and using null for everything else.
285	Note that enough space for a full SVE dump was originally allocated
286	for base. */
287
288	header->flags \|= SVE_PT_REGS_SVE;
289	header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
290
291	memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
292	sizeof (uint32_t));
293	memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
294	sizeof (uint32_t));
295
296	for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
297	{
298	memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
299	sizeof (__int128_t));
300	}
301	}
302
303	/* Replace the kernel values with those from reg_buf. */
304
305	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
306	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
307	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
308	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
309
310	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
311	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
312	reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
313	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
314
315	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
316	reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
317	base + SVE_PT_SVE_FFR_OFFSET (vq));
318	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
319	reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
320	base + SVE_PT_SVE_FPSR_OFFSET (vq));
321	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
322	reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
323	base + SVE_PT_SVE_FPCR_OFFSET (vq));
324
325	}