arch/s390/lib/uaccess_mvcos.c

   1 /*
   2  *  Optimized user space space access functions based on mvcos.
   3  *
   4  *    Copyright IBM Corp. 2006
   5  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
   6  *               Gerald Schaefer (gerald.schaefer@de.ibm.com)
   7  */
   8
   9 #include <linux/jump_label.h>
  10 #include <linux/errno.h>
  11 #include <linux/init.h>
  12 #include <linux/mm.h>
  13 #include <asm/facility.h>
  14 #include <asm/uaccess.h>
  15 #include <asm/futex.h>
  16 #include "uaccess.h"
  17
  18 #ifndef CONFIG_64BIT
  19 #define AHI     "ahi"
  20 #define ALR     "alr"
  21 #define CLR     "clr"
  22 #define LHI     "lhi"
  23 #define SLR     "slr"
  24 #else
  25 #define AHI     "aghi"
  26 #define ALR     "algr"
  27 #define CLR     "clgr"
  28 #define LHI     "lghi"
  29 #define SLR     "slgr"
  30 #endif
  31
  32 static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE;
  33
  34 static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
  35                                                  unsigned long size)
  36 {
  37         register unsigned long reg0 asm("0") = 0x81UL;
  38         unsigned long tmp1, tmp2;
  39
  40         tmp1 = -4096UL;
  41         asm volatile(
  42                 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
  43                 "9: jz    7f\n"
  44                 "1:"ALR"  %0,%3\n"
  45                 "  "SLR"  %1,%3\n"
  46                 "  "SLR"  %2,%3\n"
  47                 "   j     0b\n"
  48                 "2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
  49                 "   nr    %4,%3\n"      /* %4 = (ptr + 4095) & -4096 */
  50                 "  "SLR"  %4,%1\n"
  51                 "  "CLR"  %0,%4\n"      /* copy crosses next page boundary? */
  52                 "   jnh   4f\n"
  53                 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
  54                 "10:"SLR"  %0,%4\n"
  55                 "  "ALR"  %2,%4\n"
  56                 "4:"LHI"  %4,-1\n"
  57                 "  "ALR"  %4,%0\n"      /* copy remaining size, subtract 1 */
  58                 "   bras  %3,6f\n"      /* memset loop */
  59                 "   xc    0(1,%2),0(%2)\n"
  60                 "5: xc    0(256,%2),0(%2)\n"
  61                 "   la    %2,256(%2)\n"
  62                 "6:"AHI"  %4,-256\n"
  63                 "   jnm   5b\n"
  64                 "   ex    %4,0(%3)\n"
  65                 "   j     8f\n"
  66                 "7:"SLR"  %0,%0\n"
  67                 "8: \n"
  68                 EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b)
  69                 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
  70                 : "d" (reg0) : "cc", "memory");
  71         return size;
  72 }
  73
  74 unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
  75 {
  76         if (static_key_true(&have_mvcos))
  77                 return copy_from_user_mvcos(to, from, n);
  78         return copy_from_user_pt(to, from, n);
  79 }
  80 EXPORT_SYMBOL(__copy_from_user);
  81
  82 static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
  83                                                unsigned long size)
  84 {
  85         register unsigned long reg0 asm("0") = 0x810000UL;
  86         unsigned long tmp1, tmp2;
  87
  88         tmp1 = -4096UL;
  89         asm volatile(
  90                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
  91                 "6: jz    4f\n"
  92                 "1:"ALR"  %0,%3\n"
  93                 "  "SLR"  %1,%3\n"
  94                 "  "SLR"  %2,%3\n"
  95                 "   j     0b\n"
  96                 "2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
  97                 "   nr    %4,%3\n"      /* %4 = (ptr + 4095) & -4096 */
  98                 "  "SLR"  %4,%1\n"
  99                 "  "CLR"  %0,%4\n"      /* copy crosses next page boundary? */
 100                 "   jnh   5f\n"
 101                 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
 102                 "7:"SLR"  %0,%4\n"
 103                 "   j     5f\n"
 104                 "4:"SLR"  %0,%0\n"
 105                 "5: \n"
 106                 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
 107                 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
 108                 : "d" (reg0) : "cc", "memory");
 109         return size;
 110 }
 111
 112 unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
 113 {
 114         if (static_key_true(&have_mvcos))
 115                 return copy_to_user_mvcos(to, from, n);
 116         return copy_to_user_pt(to, from, n);
 117 }
 118 EXPORT_SYMBOL(__copy_to_user);
 119
 120 static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
 121                                                unsigned long size)
 122 {
 123         register unsigned long reg0 asm("0") = 0x810081UL;
 124         unsigned long tmp1, tmp2;
 125
 126         tmp1 = -4096UL;
 127         /* FIXME: copy with reduced length. */
 128         asm volatile(
 129                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
 130                 "   jz    2f\n"
 131                 "1:"ALR"  %0,%3\n"
 132                 "  "SLR"  %1,%3\n"
 133                 "  "SLR"  %2,%3\n"
 134                 "   j     0b\n"
 135                 "2:"SLR"  %0,%0\n"
 136                 "3: \n"
 137                 EX_TABLE(0b,3b)
 138                 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
 139                 : "d" (reg0) : "cc", "memory");
 140         return size;
 141 }
 142
 143 unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n)
 144 {
 145         if (static_key_true(&have_mvcos))
 146                 return copy_in_user_mvcos(to, from, n);
 147         return copy_in_user_pt(to, from, n);
 148 }
 149 EXPORT_SYMBOL(__copy_in_user);
 150
 151 static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
 152 {
 153         register unsigned long reg0 asm("0") = 0x810000UL;
 154         unsigned long tmp1, tmp2;
 155
 156         tmp1 = -4096UL;
 157         asm volatile(
 158                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
 159                 "   jz    4f\n"
 160                 "1:"ALR"  %0,%2\n"
 161                 "  "SLR"  %1,%2\n"
 162                 "   j     0b\n"
 163                 "2: la    %3,4095(%1)\n"/* %4 = to + 4095 */
 164                 "   nr    %3,%2\n"      /* %4 = (to + 4095) & -4096 */
 165                 "  "SLR"  %3,%1\n"
 166                 "  "CLR"  %0,%3\n"      /* copy crosses next page boundary? */
 167                 "   jnh   5f\n"
 168                 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
 169                 "  "SLR"  %0,%3\n"
 170                 "   j     5f\n"
 171                 "4:"SLR"  %0,%0\n"
 172                 "5: \n"
 173                 EX_TABLE(0b,2b) EX_TABLE(3b,5b)
 174                 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
 175                 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
 176         return size;
 177 }
 178
 179 unsigned long __clear_user(void __user *to, unsigned long size)
 180 {
 181         if (static_key_true(&have_mvcos))
 182                 return clear_user_mvcos(to, size);
 183         return clear_user_pt(to, size);
 184 }
 185 EXPORT_SYMBOL(__clear_user);
 186
 187 static inline unsigned long strnlen_user_mvcos(const char __user *src,
 188                                                unsigned long count)
 189 {
 190         unsigned long done, len, offset, len_str;
 191         char buf[256];
 192
 193         done = 0;
 194         do {
 195                 offset = (unsigned long)src & ~PAGE_MASK;
 196                 len = min(256UL, PAGE_SIZE - offset);
 197                 len = min(count - done, len);
 198                 if (copy_from_user_mvcos(buf, src, len))
 199                         return 0;
 200                 len_str = strnlen(buf, len);
 201                 done += len_str;
 202                 src += len_str;
 203         } while ((len_str == len) && (done < count));
 204         return done + 1;
 205 }
 206
 207 unsigned long __strnlen_user(const char __user *src, unsigned long count)
 208 {
 209         if (static_key_true(&have_mvcos))
 210                 return strnlen_user_mvcos(src, count);
 211         return strnlen_user_pt(src, count);
 212 }
 213 EXPORT_SYMBOL(__strnlen_user);
 214
 215 static inline long strncpy_from_user_mvcos(char *dst, const char __user *src,
 216                                            long count)
 217 {
 218         unsigned long done, len, offset, len_str;
 219
 220         if (unlikely(count <= 0))
 221                 return 0;
 222         done = 0;
 223         do {
 224                 offset = (unsigned long)src & ~PAGE_MASK;
 225                 len = min(count - done, PAGE_SIZE - offset);
 226                 if (copy_from_user_mvcos(dst, src, len))
 227                         return -EFAULT;
 228                 len_str = strnlen(dst, len);
 229                 done += len_str;
 230                 src += len_str;
 231                 dst += len_str;
 232         } while ((len_str == len) && (done < count));
 233         return done;
 234 }
 235
 236 long __strncpy_from_user(char *dst, const char __user *src, long count)
 237 {
 238         if (static_key_true(&have_mvcos))
 239                 return strncpy_from_user_mvcos(dst, src, count);
 240         return strncpy_from_user_pt(dst, src, count);
 241 }
 242 EXPORT_SYMBOL(__strncpy_from_user);
 243
 244 /*
 245  * The uaccess page tabe walk variant can be enforced with the "uaccesspt"
 246  * kernel parameter. This is mainly for debugging purposes.
 247  */
 248 static int force_uaccess_pt __initdata;
 249
 250 static int __init parse_uaccess_pt(char *__unused)
 251 {
 252         force_uaccess_pt = 1;
 253         return 0;
 254 }
 255 early_param("uaccesspt", parse_uaccess_pt);
 256
 257 static int __init uaccess_init(void)
 258 {
 259         if (IS_ENABLED(CONFIG_32BIT) || force_uaccess_pt || !test_facility(27))
 260                 static_key_slow_dec(&have_mvcos);
 261         return 0;
 262 }
 263 early_initcall(uaccess_init);