powerpc: Speed up clear_page by unrolling it

[deliverable/linux.git] / fs / namei.c

diff --git a/fs/namei.c b/fs/namei.c
index a996bb48dfabf4f645dced56f527ed3d5568bb2c..215e44254c5328c1a992db483da80e27a1dbed6c 100644 (file)
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -34,6 +34,7 @@
 #include <linux/device_cgroup.h>
 #include <linux/fs_struct.h>
 #include <linux/posix_acl.h>
+#include <linux/hash.h>
 #include <asm/uaccess.h>
 
 #include "internal.h"
@@ -643,24 +644,22 @@ static int complete_walk(struct nameidata *nd)
 
 static __always_inline void set_root(struct nameidata *nd)
 {
-       if (!nd->root.mnt)
-               get_fs_root(current->fs, &nd->root);
+       get_fs_root(current->fs, &nd->root);
 }
 
 static int link_path_walk(const char *, struct nameidata *);
 
-static __always_inline void set_root_rcu(struct nameidata *nd)
+static __always_inline unsigned set_root_rcu(struct nameidata *nd)
 {
-       if (!nd->root.mnt) {
-               struct fs_struct *fs = current->fs;
-               unsigned seq;
+       struct fs_struct *fs = current->fs;
+       unsigned seq, res;
 
-               do {
-                       seq = read_seqcount_begin(&fs->seq);
-                       nd->root = fs->root;
-                       nd->seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
-               } while (read_seqcount_retry(&fs->seq, seq));
-       }
+       do {
+               seq = read_seqcount_begin(&fs->seq);
+               nd->root = fs->root;
+               res = __read_seqcount_begin(&nd->root.dentry->d_seq);
+       } while (read_seqcount_retry(&fs->seq, seq));
+       return res;
 }
 
 static void path_put_conditional(struct path *path, struct nameidata *nd)
@@ -860,7 +859,8 @@ follow_link(struct path *link, struct nameidata *nd, void **p)
                        return PTR_ERR(s);
                }
                if (*s == '/') {
-                       set_root(nd);
+                       if (!nd->root.mnt)
+                               set_root(nd);
                        path_put(&nd->path);
                        nd->path = nd->root;
                        path_get(&nd->root);
@@ -1137,13 +1137,15 @@ static bool __follow_mount_rcu(struct nameidata *nd, struct path *path,
                 */
                *inode = path->dentry->d_inode;
        }
-       return read_seqretry(&mount_lock, nd->m_seq) &&
+       return !read_seqretry(&mount_lock, nd->m_seq) &&
                !(path->dentry->d_flags & DCACHE_NEED_AUTOMOUNT);
 }
 
 static int follow_dotdot_rcu(struct nameidata *nd)
 {
-       set_root_rcu(nd);
+       struct inode *inode = nd->inode;
+       if (!nd->root.mnt)
+               set_root_rcu(nd);
 
        while (1) {
                if (nd->path.dentry == nd->root.dentry &&
@@ -1155,6 +1157,7 @@ static int follow_dotdot_rcu(struct nameidata *nd)
                        struct dentry *parent = old->d_parent;
                        unsigned seq;
 
+                       inode = parent->d_inode;
                        seq = read_seqcount_begin(&parent->d_seq);
                        if (read_seqcount_retry(&old->d_seq, nd->seq))
                                goto failed;
@@ -1164,6 +1167,7 @@ static int follow_dotdot_rcu(struct nameidata *nd)
                }
                if (!follow_up_rcu(&nd->path))
                        break;
+               inode = nd->path.dentry->d_inode;
                nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
        }
        while (d_mountpoint(nd->path.dentry)) {
@@ -1173,11 +1177,12 @@ static int follow_dotdot_rcu(struct nameidata *nd)
                        break;
                nd->path.mnt = &mounted->mnt;
                nd->path.dentry = mounted->mnt.mnt_root;
+               inode = nd->path.dentry->d_inode;
                nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
-               if (!read_seqretry(&mount_lock, nd->m_seq))
+               if (read_seqretry(&mount_lock, nd->m_seq))
                        goto failed;
        }
-       nd->inode = nd->path.dentry->d_inode;
+       nd->inode = inode;
        return 0;
 
 failed:
@@ -1256,7 +1261,8 @@ static void follow_mount(struct path *path)
 
 static void follow_dotdot(struct nameidata *nd)
 {
-       set_root(nd);
+       if (!nd->root.mnt)
+               set_root(nd);
 
        while(1) {
                struct dentry *old = nd->path.dentry;
@@ -1634,8 +1640,7 @@ static inline int nested_symlink(struct path *path, struct nameidata *nd)
 
 static inline unsigned int fold_hash(unsigned long hash)
 {
-       hash += hash >> (8*sizeof(int));
-       return hash;
+       return hash_64(hash, 32);
 }
 
 #else  /* 32-bit case */
@@ -1669,13 +1674,14 @@ EXPORT_SYMBOL(full_name_hash);
 
 /*
  * Calculate the length and hash of the path component, and
- * return the length of the component;
+ * fill in the qstr. return the "len" as the result.
  */
-static inline unsigned long hash_name(const char *name, unsigned int *hashp)
+static inline unsigned long hash_name(const char *name, struct qstr *res)
 {
        unsigned long a, b, adata, bdata, mask, hash, len;
        const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
 
+       res->name = name;
        hash = a = 0;
        len = -sizeof(unsigned long);
        do {
@@ -1691,9 +1697,10 @@ static inline unsigned long hash_name(const char *name, unsigned int *hashp)
        mask = create_zero_mask(adata | bdata);
 
        hash += a & zero_bytemask(mask);
-       *hashp = fold_hash(hash);
+       len += find_zero(mask);
+       res->hash_len = hashlen_create(fold_hash(hash), len);
 
-       return len + find_zero(mask);
+       return len;
 }
 
 #else
@@ -1711,18 +1718,19 @@ EXPORT_SYMBOL(full_name_hash);
  * We know there's a real path component here of at least
  * one character.
  */
-static inline unsigned long hash_name(const char *name, unsigned int *hashp)
+static inline long hash_name(const char *name, struct qstr *res)
 {
        unsigned long hash = init_name_hash();
        unsigned long len = 0, c;
 
+       res->name = name;
        c = (unsigned char)*name;
        do {
                len++;
                hash = partial_name_hash(c, hash);
                c = (unsigned char)name[len];
        } while (c && c != '/');
-       *hashp = end_name_hash(hash);
+       res->hash_len = hashlen_create(end_name_hash(hash), len);
        return len;
 }
 
@@ -1756,9 +1764,7 @@ static int link_path_walk(const char *name, struct nameidata *nd)
                if (err)
                        break;
 
-               len = hash_name(name, &this.hash);
-               this.name = name;
-               this.len = len;
+               len = hash_name(name, &this);
 
                type = LAST_NORM;
                if (name[0] == '.') switch (len) {
@@ -1852,7 +1858,7 @@ static int path_init(int dfd, const char *name, unsigned int flags,
        if (*name=='/') {
                if (flags & LOOKUP_RCU) {
                        rcu_read_lock();
-                       set_root_rcu(nd);
+                       nd->seq = set_root_rcu(nd);
                } else {
                        set_root(nd);
                        path_get(&nd->root);
@@ -1903,7 +1909,14 @@ static int path_init(int dfd, const char *name, unsigned int flags,
        }
 
        nd->inode = nd->path.dentry->d_inode;
-       return 0;
+       if (!(flags & LOOKUP_RCU))
+               return 0;
+       if (likely(!read_seqcount_retry(&nd->path.dentry->d_seq, nd->seq)))
+               return 0;
+       if (!(nd->flags & LOOKUP_ROOT))
+               nd->root.mnt = NULL;
+       rcu_read_unlock();
+       return -ECHILD;
 }
 
 static inline int lookup_last(struct nameidata *nd, struct path *path)