fs.path optimizations since these functions are called so frequently

fs/base.py

@@ -785,7 +785,7 @@ class FS(object):
         if size is None:
             raise OperationFailedError("get size of resource", path)
         return size
-
+    
     def copy(self, src, dst, overwrite=False, chunk_size=1024*64):
         """Copies a file from src to dst.
 
@@ -836,7 +836,7 @@ class FS(object):
     def _shutil_movefile(cls, src_syspath, dst_syspath):
         shutil.move(src_syspath, dst_syspath)
         
-
+    
     def move(self, src, dst, overwrite=False, chunk_size=16384):
         """moves a file from one location to another.
 
@@ -872,7 +872,7 @@ class FS(object):
                 pass
         self.copy(src, dst, overwrite=overwrite, chunk_size=chunk_size)
         self.remove(src)
-
+    
     def movedir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
         """moves a directory from one location to another.
 
@@ -931,7 +931,7 @@ class FS(object):
                 movefile(src_filename, dst_filename, overwrite=overwrite, chunk_size=chunk_size)
 
             self.removedir(dirname)
-
+    
     def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
         """copies a directory from one location to another.
 

fs/memoryfs.py

@@ -32,13 +32,13 @@ class MemoryFile(object):
     def seek_and_lock(f):
         def deco(self, *args, **kwargs):
             try:
-                self.lock.acquire()
+                self._lock.acquire()
                 self.mem_file.seek(self.pos)
                 ret = f(self, *args, **kwargs)
                 self.pos = self.mem_file.tell()
                 return ret
             finally:                
-                self.lock.release()
+                self._lock.release()
         return deco
 
     def __init__(self, path, memory_fs, mem_file, mode, lock):
@@ -47,7 +47,7 @@ class MemoryFile(object):
         self.memory_fs = memory_fs
         self.mem_file = mem_file        
         self.mode = mode        
-        self.lock = lock
+        self._lock = lock
         
         self.pos = 0                
                 
@@ -59,7 +59,7 @@ class MemoryFile(object):
             finally:
                 lock.release()
         
-        if _check_mode(mode, 'w'):
+        elif _check_mode(mode, 'w'):
             lock.acquire()
             try:
                 self.mem_file.seek(0)
@@ -96,12 +96,17 @@ class MemoryFile(object):
     def readline(self, *args, **kwargs):
         return self.mem_file.readline(*args, **kwargs)
 
-
-    #@seek_and_lock
     def close(self):
-        if not self.closed and self.mem_file is not None:
-            self.memory_fs._on_close_memory_file(self, self.path)
-            self.closed = True            
+        do_close = False
+        self._lock.acquire()
+        try:
+            do_close = not self.closed and self.mem_file is not None
+            if do_close:
+                self.closed = True
+        finally:
+            self._lock.release()                
+        if do_close:
+            self.memory_fs._on_close_memory_file(self, self.path)            
 
     @seek_and_lock
     def read(self, size=None):
@@ -124,13 +129,13 @@ class MemoryFile(object):
     #@seek_and_lock
     def write(self, data):        
         self.memory_fs._on_modify_memory_file(self.path)
-        self.lock.acquire()
+        self._lock.acquire()
         try:
             self.mem_file.seek(self.pos)
             self.mem_file.write(data)
             self.pos = self.mem_file.tell()
         finally:
-            self.lock.release()
+            self._lock.release()
 
     @seek_and_lock
     def writelines(self, *args, **kwargs):        
@@ -385,6 +390,7 @@ class MemoryFS(FS):
 
     @synchronize
     def open(self, path, mode="r", **kwargs):
+        path = normpath(path)
         filepath, filename = pathsplit(path)
         parent_dir_entry = self._get_dir_entry(filepath)
 
@@ -432,13 +438,12 @@ class MemoryFS(FS):
             raise ResourceInvalidError(path,msg="That's a directory, not a file: %(path)s")
 
         pathname, dirname = pathsplit(path)
-
         parent_dir = self._get_dir_entry(pathname)
-
         del parent_dir.contents[dirname]
 
     @synchronize
     def removedir(self, path, recursive=False, force=False):
+        path = normpath(path)
         dir_entry = self._get_dir_entry(path)
 
         if dir_entry is None:
@@ -462,7 +467,9 @@ class MemoryFS(FS):
 
     @synchronize
     def rename(self, src, dst):
-        src_dir,src_name = pathsplit(src)
+        src = normpath(src)
+        dst = normpath(dst)
+        src_dir, src_name = pathsplit(src)
         src_entry = self._get_dir_entry(src)
         if src_entry is None:
             raise ResourceNotFoundError(src)
@@ -504,13 +511,15 @@ class MemoryFS(FS):
     @synchronize
     def _on_close_memory_file(self, open_file, path):
         dir_entry = self._get_dir_entry(path)
-        dir_entry.open_files.remove(open_file)        
+        if dir_entry is not None:
+            dir_entry.open_files.remove(open_file)        
                 
         
     @synchronize
     def _on_modify_memory_file(self, path):
         dir_entry = self._get_dir_entry(path)
-        dir_entry.modified_time = datetime.datetime.now()        
+        if dir_entry is not None:
+            dir_entry.modified_time = datetime.datetime.now()        
 
     @synchronize
     def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):

fs/path.py

@@ -37,25 +37,21 @@ def normpath(path):
     if not path:
         return path
     components = []
-    for comp in path.replace('\\','/').split("/"):
-        if not comp or comp == ".":
-            pass
-        elif comp == "..":
+    append = components.append    
+    for comp in [c for c in path.replace('\\','/').split("/") if c not in ('', '.')]:        
+        if comp == "..":
             try:
                 components.pop()
             except IndexError:
                 err = "too many backrefs in path '%s'" % (path,)
                 raise ValueError(err)
         else:
-            components.append(comp)
-    if path[0] in "\\/":
+            append(comp)
+    if path[0] in '\\/':
         if not components:
-            components = [""]
-        components.insert(0, "")
-    if isinstance(path, unicode):
-        return u"/".join(components)
-    else:
-        return '/'.join(components)
+            append("")
+        components.insert(0, "")    
+    return "/".join(components)    
 
 
 def iteratepath(path, numsplits=None):
@@ -69,9 +65,9 @@ def iteratepath(path, numsplits=None):
     if not path:
         return []
     if numsplits == None:
-        return map(None, path.split('/'))
+        return path.split('/')
     else:
-        return map(None, path.split('/', numsplits))
+        return path.split('/', numsplits)
         
 def recursepath(path, reverse=False):
     """Returns intermediate paths from the root to the given path
@@ -84,11 +80,13 @@ def recursepath(path, reverse=False):
     """
     if reverse:
         paths = []
+        append = paths.append
         path = abspath(normpath(path)).rstrip("/")
         while path:
-            paths.append(path)
+            append(path)
             path = dirname(path).rstrip("/")
-        return paths + [u"/"]
+        paths.append(u"/")
+        return paths
     else:   
         paths = [u""] + list(iteratepath(path))
         return [u"/"] + [u'/'.join(paths[:i+1]) for i in xrange(1,len(paths))]
@@ -100,8 +98,6 @@ def abspath(path):
     adds a leading '/' character if the path doesn't already have one.
 
     """
-    if not path:
-        return u'/'
     if not path.startswith('/'):
         return u'/' + path
     return path
@@ -138,7 +134,7 @@ def pathjoin(*paths):
 
     """
     absolute = False
-    relpaths = []
+    relpaths = []    
     for p in paths:
         if p:
              if p[0] in '\\/':
@@ -173,10 +169,10 @@ def pathsplit(path):
     ('/foo/bar', 'baz')
 
     """
-    split = normpath(path).rsplit('/', 1)
-    if len(split) == 1:
-        return (u'', split[0])
-    return split[0] or '/', split[1]
+    if '/' not in path:
+        return ('', path)
+    split = path.rsplit('/', 1)
+    return (split[0] or '/', split[1])
 
 # Allow pathsplit() to be used as fs.path.split()
 split = pathsplit
@@ -211,14 +207,14 @@ def isdotfile(path):
     >>> isdotfile('.baz')
     True
     
-    >>> isdotfile('foo/bar/.baz')
+    >>> isdotfile('foo/bar/baz')
     True
     
-    >>> isdotfile('foo/bar.baz')
+    >>> isdotfile('foo/bar.baz').
     False
     
     """
-    return pathsplit(path)[-1].startswith('.')
+    return basename(path).startswith('.')
 
 def dirname(path):
     """Returns the parent directory of a path.
@@ -232,8 +228,10 @@ def dirname(path):
     'foo/bar'
 
     """
-    return pathsplit(path)[0]
-
+    if '/' not in path:
+        return ''
+    return path.rsplit('/', 1)[0]
+    
 
 def basename(path):
     """Returns the basename of the resource referenced by a path.
@@ -247,7 +245,9 @@ def basename(path):
     'baz'
 
     """
-    return pathsplit(path)[1]
+    if '/' not in path:
+        return path
+    return path.rsplit('/', 1)[-1]
 
 
 def issamedir(path1, path2):
@@ -262,7 +262,7 @@ def issamedir(path1, path2):
     False
 
     """
-    return pathsplit(normpath(path1))[0] == pathsplit(normpath(path2))[0]
+    return dirname(normpath(path1)) == dirname(normpath(path2))
 
 def isbase(path1, path2):
     p1 = forcedir(abspath(path1))

fs/tests/test_path.py

@@ -102,6 +102,37 @@ class TestPathFunctions(unittest.TestCase):
         self.assertEquals(recursepath("/hello/world/",reverse=True),["/hello/world","/hello","/"])
         self.assertEquals(recursepath("hello",reverse=True),["/hello","/"])
         self.assertEquals(recursepath("",reverse=True),["/"])
+        
+    def test_isdotfile(self):
+        for path in ['.foo',
+                     '.svn',
+                     'foo/.svn',
+                     'foo/bar/.svn',
+                     '/foo/.bar']:
+            self.assert_(isdotfile(path))
+        
+        for path in ['asfoo',
+                     'df.svn',
+                     'foo/er.svn',
+                     'foo/bar/test.txt',
+                     '/foo/bar']:
+            self.assertFalse(isdotfile(path))
+            
+    def test_dirname(self):
+        tests = [('foo', ''),
+                 ('foo/bar', 'foo'),
+                 ('foo/bar/baz', 'foo/bar'),
+                 ('/', '')]
+        for path, test_dirname in tests:
+            self.assertEqual(dirname(path), test_dirname)
+            
+    def test_basename(self):
+        tests = [('foo', 'foo'),
+                 ('foo/bar', 'bar'),
+                 ('foo/bar/baz', 'baz'),
+                 ('/', '')]
+        for path, test_basename in tests:
+            self.assertEqual(basename(path), test_basename)
 
 
 class Test_PathMap(unittest.TestCase):