RemoteFileBuffer: implement on-demand loading of remote data.

Closes Issue #33, thanks to Marek Palatinus for the implementation.

fs/tests/test_remote.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 
   fs.tests.test_remote:  testcases for FS remote support utilities
@@ -14,11 +16,181 @@ import sys
 
 from fs.remote import *
 
+from fs import SEEK_END
 from fs.wrapfs import WrapFS, wrap_fs_methods
 from fs.tempfs import TempFS
 from fs.path import *
 from fs.local_functools import wraps
 
+class RemoteTempFS(TempFS):
+    '''
+        Simple filesystem implementing setfilecontents
+        for RemoteFileBuffer tests
+    '''
+    def open(self, path, mode='rb', write_on_flush=True):
+        if 'a' in mode or 'r' in mode or '+' in mode:
+            f = super(RemoteTempFS, self).open(path, 'rb')
+        else:
+            f = None
+        
+        return RemoteFileBuffer(self, path, mode, f,
+                    write_on_flush=write_on_flush)
+        
+    def setcontents(self, path, content):
+        f = super(RemoteTempFS, self).open(path, 'wb')
+        if getattr(content, 'read', False):
+            f.write(content.read())
+        else:
+            f.write(content)
+        f.close()
+
+class TestRemoteFileBuffer(unittest.TestCase, FSTestCases, ThreadingTestCases):
+    class FakeException(Exception): pass
+    
+    def setUp(self):
+        self.fs = RemoteTempFS()
+        self.original_setcontents = self.fs.setcontents
+
+    def tearDown(self):
+        self.fs.close()
+
+    def fake_setcontents(self, path, content):
+        ''' Fake replacement for RemoteTempFS setcontents() '''
+        raise self.FakeException("setcontents should not be called here!")
+    
+    def fakeOn(self):
+        '''
+            Turn on fake_setcontents(). When setcontents on RemoteTempFS
+            is called, FakeException is raised and nothing is stored.
+        '''
+        self.original_setcontents = self.fs.setcontents
+        self.fs.setcontents = self.fake_setcontents
+        
+    def fakeOff(self):
+        ''' Switch off fake_setcontents(). '''
+        self.fs.setcontents = self.original_setcontents
+        
+    def test_ondemand(self):
+        '''
+            Tests on-demand loading of remote content in RemoteFileBuffer
+        '''    
+        contents = "Tristatricettri stribrnych strikacek strikalo" + \
+                   "pres tristatricettri stribrnych strech."
+        f = self.fs.open('test.txt', 'wb')
+        f.write(contents)
+        f.close()
+        
+        # During following tests, no setcontents() should be called.
+        self.fakeOn()
+        
+        f = self.fs.open('test.txt', 'rb')
+        self.assertEquals(f.read(10), contents[:10])
+        f.file.seek(0, SEEK_END)
+        self.assertEquals(f._rfile.tell(), 10)
+        f.seek(20)
+        self.assertEquals(f.tell(), 20)
+        self.assertEquals(f._rfile.tell(), 20)
+        f.seek(0, SEEK_END)
+        self.assertEquals(f._rfile.tell(), len(contents))
+        f.close()
+        
+        f = self.fs.open('test.txt', 'ab')
+        self.assertEquals(f.tell(), len(contents))
+        f.close()
+        
+        self.fakeOff()
+        
+        # Writing over the rfile edge
+        f = self.fs.open('test.txt', 'wb+')
+        self.assertEquals(f.tell(), 0)
+        f.seek(len(contents) - 5)
+        # Last 5 characters not loaded from remote file  
+        self.assertEquals(f._rfile.tell(), len(contents) - 5)
+        # Confirm that last 5 characters are still in rfile buffer
+        self.assertEquals(f._rfile.read(), contents[-5:])
+        # Rollback position 5 characters before eof
+        f._rfile.seek(len(contents[:-5]))
+        # Write 10 new characters (will make contents longer for 5 chars)
+        f.write(u'1234567890')
+        # We are on the end of file (and buffer not serve anything anymore)
+        self.assertEquals(f.read(), '')
+        f.close()
+        
+        self.fakeOn()
+        
+        # Check if we wrote everything OK from
+        # previous writing over the remote buffer edge
+        f = self.fs.open('test.txt', 'rb')
+        self.assertEquals(f.read(), contents[:-5] + u'1234567890')
+        f.close()
+    
+    def test_writeonflush(self):
+        '''
+            Test 'write_on_flush' switch of RemoteFileBuffer.
+            When True, flush() should call setcontents and store
+            to remote destination.
+            When False, setcontents should be called only on close().
+        '''
+        self.fakeOn()
+        f = self.fs.open('test.txt', 'wb', write_on_flush=True)
+        f.write('Sample text')
+        self.assertRaises(self.FakeException, f.flush)
+        f.write('Second sample text')
+        self.assertRaises(self.FakeException, f.close)
+        
+        f = self.fs.open('test.txt', 'wb', write_on_flush=False)
+        f.write('Sample text')
+        # FakeException is not raised, because setcontents is not called
+        f.flush()
+        f.write('Second sample text')
+        self.assertRaises(self.FakeException, f.close)
+        
+    def test_flush_and_continue(self):
+        '''
+            This tests if partially loaded remote buffer can be flushed
+            back to remote destination and opened file is still
+            in good condition.        
+        '''
+        contents = "Zlutoucky kun upel dabelske ody."
+        contents2 = 'Ententyky dva spaliky cert vyletel z elektriky'
+                
+        f = self.fs.open('test.txt', 'wb')
+        f.write(contents)
+        f.close()
+
+        f = self.fs.open('test.txt', 'rb+')
+        # Check if we read just 10 characters
+        self.assertEquals(f.read(10), contents[:10])
+        self.assertEquals(f._rfile.tell(), 10)
+        # Write garbage to file to mark it as _changed
+        f.write('x')
+        # This should read the rest of file and store file back to again.
+        f.flush()
+        f.seek(0)
+        # Try if we have unocrrupted file locally...
+        self.assertEquals(f.read(), contents[:10] + 'x' + contents[11:])
+        f.close()
+        
+        # And if we have uncorrupted file also on storage
+        f = self.fs.open('test.txt', 'rb')
+        self.assertEquals(f.read(), contents[:10] + 'x' + contents[11:])
+        f.close()
+        
+        # Now try it again, but write garbage behind edge of remote file
+        f = self.fs.open('test.txt', 'rb+')
+        self.assertEquals(f.read(10), contents[:10])
+        # Write garbage to file to mark it as _changed
+        f.write(contents2)
+        f.flush()
+        f.seek(0)
+        # Try if we have unocrrupted file locally...
+        self.assertEquals(f.read(), contents[:10] + contents2)
+        f.close()
+        
+        # And if we have uncorrupted file also on storage
+        f = self.fs.open('test.txt', 'rb')
+        self.assertEquals(f.read(), contents[:10] + contents2)
+        f.close()
         
 class TestCacheFS(unittest.TestCase,FSTestCases,ThreadingTestCases):
     """Test simple operation of CacheFS"""
@@ -129,4 +301,5 @@ class TestConnectionManagerFS_disconnect(TestConnectionManagerFS):
         self.fs.close()
         sys.setcheckinterval(self._check_interval)
 
-
+if __name__ == '__main__':
+    unittest.main()