lib/raid6/sse2.c

   1 /* -*- linux-c -*- ------------------------------------------------------- *
   2  *
   3  *   Copyright 2002 H. Peter Anvin - All Rights Reserved
   4  *
   5  *   This program is free software; you can redistribute it and/or modify
   6  *   it under the terms of the GNU General Public License as published by
   7  *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
   8  *   Boston MA 02111-1307, USA; either version 2 of the License, or
   9  *   (at your option) any later version; incorporated herein by reference.
  10  *
  11  * ----------------------------------------------------------------------- */
  12
  13 /*
  14  * raid6/sse2.c
  15  *
  16  * SSE-2 implementation of RAID-6 syndrome functions
  17  *
  18  */
  19
  20 #include <linux/raid/pq.h>
  21 #include "x86.h"
  22
  23 static const struct raid6_sse_constants {
  24         u64 x1d[2];
  25 } raid6_sse_constants  __attribute__((aligned(16))) = {
  26         { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
  27 };
  28
  29 static int raid6_have_sse2(void)
  30 {
  31         /* Not really boot_cpu but "all_cpus" */
  32         return boot_cpu_has(X86_FEATURE_MMX) &&
  33                 boot_cpu_has(X86_FEATURE_FXSR) &&
  34                 boot_cpu_has(X86_FEATURE_XMM) &&
  35                 boot_cpu_has(X86_FEATURE_XMM2);
  36 }
  37
  38 /*
  39  * Plain SSE2 implementation
  40  */
  41 static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
  42 {
  43         u8 **dptr = (u8 **)ptrs;
  44         u8 *p, *q;
  45         int d, z, z0;
  46
  47         z0 = disks - 3;         /* Highest data disk */
  48         p = dptr[z0+1];         /* XOR parity */
  49         q = dptr[z0+2];         /* RS syndrome */
  50
  51         kernel_fpu_begin();
  52
  53         asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
  54         asm volatile("pxor %xmm5,%xmm5");       /* Zero temp */
  55
  56         for ( d = 0 ; d < bytes ; d += 16 ) {
  57                 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
  58                 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
  59                 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
  60                 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
  61                 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
  62                 for ( z = z0-2 ; z >= 0 ; z-- ) {
  63                         asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
  64                         asm volatile("pcmpgtb %xmm4,%xmm5");
  65                         asm volatile("paddb %xmm4,%xmm4");
  66                         asm volatile("pand %xmm0,%xmm5");
  67                         asm volatile("pxor %xmm5,%xmm4");
  68                         asm volatile("pxor %xmm5,%xmm5");
  69                         asm volatile("pxor %xmm6,%xmm2");
  70                         asm volatile("pxor %xmm6,%xmm4");
  71                         asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
  72                 }
  73                 asm volatile("pcmpgtb %xmm4,%xmm5");
  74                 asm volatile("paddb %xmm4,%xmm4");
  75                 asm volatile("pand %xmm0,%xmm5");
  76                 asm volatile("pxor %xmm5,%xmm4");
  77                 asm volatile("pxor %xmm5,%xmm5");
  78                 asm volatile("pxor %xmm6,%xmm2");
  79                 asm volatile("pxor %xmm6,%xmm4");
  80
  81                 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
  82                 asm volatile("pxor %xmm2,%xmm2");
  83                 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
  84                 asm volatile("pxor %xmm4,%xmm4");
  85         }
  86
  87         asm volatile("sfence" : : : "memory");
  88         kernel_fpu_end();
  89 }
  90
  91 const struct raid6_calls raid6_sse2x1 = {
  92         raid6_sse21_gen_syndrome,
  93         NULL,                   /* XOR not yet implemented */
  94         raid6_have_sse2,
  95         "sse2x1",
  96         1                       /* Has cache hints */
  97 };
  98
  99 /*
 100  * Unrolled-by-2 SSE2 implementation
 101  */
 102 static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
 103 {
 104         u8 **dptr = (u8 **)ptrs;
 105         u8 *p, *q;
 106         int d, z, z0;
 107
 108         z0 = disks - 3;         /* Highest data disk */
 109         p = dptr[z0+1];         /* XOR parity */
 110         q = dptr[z0+2];         /* RS syndrome */
 111
 112         kernel_fpu_begin();
 113
 114         asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
 115         asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
 116         asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
 117
 118         /* We uniformly assume a single prefetch covers at least 32 bytes */
 119         for ( d = 0 ; d < bytes ; d += 32 ) {
 120                 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
 121                 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));    /* P[0] */
 122                 asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
 123                 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
 124                 asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
 125                 for ( z = z0-1 ; z >= 0 ; z-- ) {
 126                         asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
 127                         asm volatile("pcmpgtb %xmm4,%xmm5");
 128                         asm volatile("pcmpgtb %xmm6,%xmm7");
 129                         asm volatile("paddb %xmm4,%xmm4");
 130                         asm volatile("paddb %xmm6,%xmm6");
 131                         asm volatile("pand %xmm0,%xmm5");
 132                         asm volatile("pand %xmm0,%xmm7");
 133                         asm volatile("pxor %xmm5,%xmm4");
 134                         asm volatile("pxor %xmm7,%xmm6");
 135                         asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
 136                         asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
 137                         asm volatile("pxor %xmm5,%xmm2");
 138                         asm volatile("pxor %xmm7,%xmm3");
 139                         asm volatile("pxor %xmm5,%xmm4");
 140                         asm volatile("pxor %xmm7,%xmm6");
 141                         asm volatile("pxor %xmm5,%xmm5");
 142                         asm volatile("pxor %xmm7,%xmm7");
 143                 }
 144                 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
 145                 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
 146                 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
 147                 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
 148         }
 149
 150         asm volatile("sfence" : : : "memory");
 151         kernel_fpu_end();
 152 }
 153
 154 const struct raid6_calls raid6_sse2x2 = {
 155         raid6_sse22_gen_syndrome,
 156         NULL,                   /* XOR not yet implemented */
 157         raid6_have_sse2,
 158         "sse2x2",
 159         1                       /* Has cache hints */
 160 };
 161
 162 #ifdef CONFIG_X86_64
 163
 164 /*
 165  * Unrolled-by-4 SSE2 implementation
 166  */
 167 static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
 168 {
 169         u8 **dptr = (u8 **)ptrs;
 170         u8 *p, *q;
 171         int d, z, z0;
 172
 173         z0 = disks - 3;         /* Highest data disk */
 174         p = dptr[z0+1];         /* XOR parity */
 175         q = dptr[z0+2];         /* RS syndrome */
 176
 177         kernel_fpu_begin();
 178
 179         asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
 180         asm volatile("pxor %xmm2,%xmm2");       /* P[0] */
 181         asm volatile("pxor %xmm3,%xmm3");       /* P[1] */
 182         asm volatile("pxor %xmm4,%xmm4");       /* Q[0] */
 183         asm volatile("pxor %xmm5,%xmm5");       /* Zero temp */
 184         asm volatile("pxor %xmm6,%xmm6");       /* Q[1] */
 185         asm volatile("pxor %xmm7,%xmm7");       /* Zero temp */
 186         asm volatile("pxor %xmm10,%xmm10");     /* P[2] */
 187         asm volatile("pxor %xmm11,%xmm11");     /* P[3] */
 188         asm volatile("pxor %xmm12,%xmm12");     /* Q[2] */
 189         asm volatile("pxor %xmm13,%xmm13");     /* Zero temp */
 190         asm volatile("pxor %xmm14,%xmm14");     /* Q[3] */
 191         asm volatile("pxor %xmm15,%xmm15");     /* Zero temp */
 192
 193         for ( d = 0 ; d < bytes ; d += 64 ) {
 194                 for ( z = z0 ; z >= 0 ; z-- ) {
 195                         /* The second prefetch seems to improve performance... */
 196                         asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
 197                         asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
 198                         asm volatile("pcmpgtb %xmm4,%xmm5");
 199                         asm volatile("pcmpgtb %xmm6,%xmm7");
 200                         asm volatile("pcmpgtb %xmm12,%xmm13");
 201                         asm volatile("pcmpgtb %xmm14,%xmm15");
 202                         asm volatile("paddb %xmm4,%xmm4");
 203                         asm volatile("paddb %xmm6,%xmm6");
 204                         asm volatile("paddb %xmm12,%xmm12");
 205                         asm volatile("paddb %xmm14,%xmm14");
 206                         asm volatile("pand %xmm0,%xmm5");
 207                         asm volatile("pand %xmm0,%xmm7");
 208                         asm volatile("pand %xmm0,%xmm13");
 209                         asm volatile("pand %xmm0,%xmm15");
 210                         asm volatile("pxor %xmm5,%xmm4");
 211                         asm volatile("pxor %xmm7,%xmm6");
 212                         asm volatile("pxor %xmm13,%xmm12");
 213                         asm volatile("pxor %xmm15,%xmm14");
 214                         asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
 215                         asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
 216                         asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
 217                         asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
 218                         asm volatile("pxor %xmm5,%xmm2");
 219                         asm volatile("pxor %xmm7,%xmm3");
 220                         asm volatile("pxor %xmm13,%xmm10");
 221                         asm volatile("pxor %xmm15,%xmm11");
 222                         asm volatile("pxor %xmm5,%xmm4");
 223                         asm volatile("pxor %xmm7,%xmm6");
 224                         asm volatile("pxor %xmm13,%xmm12");
 225                         asm volatile("pxor %xmm15,%xmm14");
 226                         asm volatile("pxor %xmm5,%xmm5");
 227                         asm volatile("pxor %xmm7,%xmm7");
 228                         asm volatile("pxor %xmm13,%xmm13");
 229                         asm volatile("pxor %xmm15,%xmm15");
 230                 }
 231                 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
 232                 asm volatile("pxor %xmm2,%xmm2");
 233                 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
 234                 asm volatile("pxor %xmm3,%xmm3");
 235                 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
 236                 asm volatile("pxor %xmm10,%xmm10");
 237                 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
 238                 asm volatile("pxor %xmm11,%xmm11");
 239                 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
 240                 asm volatile("pxor %xmm4,%xmm4");
 241                 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
 242                 asm volatile("pxor %xmm6,%xmm6");
 243                 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
 244                 asm volatile("pxor %xmm12,%xmm12");
 245                 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
 246                 asm volatile("pxor %xmm14,%xmm14");
 247         }
 248
 249         asm volatile("sfence" : : : "memory");
 250         kernel_fpu_end();
 251 }
 252
 253 const struct raid6_calls raid6_sse2x4 = {
 254         raid6_sse24_gen_syndrome,
 255         NULL,                   /* XOR not yet implemented */
 256         raid6_have_sse2,
 257         "sse2x4",
 258         1                       /* Has cache hints */
 259 };
 260
 261 #endif /* CONFIG_X86_64 */