Improved cUnbufferedFile

CONTRIBUTORS

@@ -271,6 +271,7 @@ Artur Skawina <skawina@geocities.com>
  for improving the font file generation in the Makefile
  for pointing out a problem with the ERR macro defined by ncurses.h
  for a patch that contained a fix for checking toFile in cCuttingThread::Action()
+ for improving cUnbufferedFile
 
 Werner Fink <werner@suse.de>
  for making I/O more robust by handling EINTR

HISTORY

@@ -4293,3 +4293,6 @@ Video Disk Recorder Revision History
   (reported by Udo Richter).
 - Fixed handling the "Setup/OSD/Menu button closes" option when set to 'yes' in
   case a replay is active (thanks to Udo Richter).
+- Improved cUnbufferedFile; USE_FADVISE is now defined in tools.c by default, so
+  if you don't want to use "fadvise" you need to comment out that line (thanks to
+  Artur Skawina).

cutter.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: cutter.c 1.12 2006/01/27 13:45:00 kls Exp $
+ * $Id: cutter.c 1.13 2006/02/04 13:40:20 kls Exp $
  */
 
 #include "cutter.h"
@@ -66,6 +66,8 @@ void cCuttingThread::Action(void)
      toFile = toFileName->Open();
      if (!fromFile || !toFile)
         return;
+     fromFile->SetReadAhead(MEGABYTE(20));
+     toFile->SetReadAhead(MEGABYTE(20));
      int Index = Mark->position;
      Mark = fromMarks.Next(Mark);
      int FileSize = 0;
@@ -90,6 +92,7 @@ void cCuttingThread::Action(void)
            if (fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) {
               if (FileNumber != CurrentFileNumber) {
                  fromFile = fromFileName->SetOffset(FileNumber, FileOffset);
+                 fromFile->SetReadAhead(MEGABYTE(20));
                  CurrentFileNumber = FileNumber;
                  }
               if (fromFile) {
@@ -122,6 +125,7 @@ void cCuttingThread::Action(void)
                     error = "toFile 1";
                     break;
                     }
+                 toFile->SetReadAhead(MEGABYTE(20));
                  FileSize = 0;
                  }
               LastIFrame = 0;
@@ -162,6 +166,7 @@ void cCuttingThread::Action(void)
                        error = "toFile 2";
                        break;
                        }
+                    toFile->SetReadAhead(MEGABYTE(20));
                     FileSize = 0;
                     }
                  }

tools.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.c 1.112 2006/01/20 14:01:28 kls Exp $
+ * $Id: tools.c 1.113 2006/02/04 14:07:30 kls Exp $
  */
 
 #include "tools.h"
@@ -1054,10 +1054,9 @@ bool cSafeFile::Close(void)
 
 // --- cUnbufferedFile -------------------------------------------------------
 
-//#define USE_FADVISE
+#define USE_FADVISE
 
-#define READ_AHEAD MEGABYTE(2)
+#define WRITE_BUFFER KILOBYTE(800)
-#define WRITE_BUFFER MEGABYTE(10)
 
 cUnbufferedFile::cUnbufferedFile(void)
 {
@@ -1073,8 +1072,17 @@ int cUnbufferedFile::Open(const char *FileName, int Flags, mode_t Mode)
 {
   Close();
   fd = open(FileName, Flags, Mode);
-  begin = end = ahead = -1;
+  curpos = 0;
+#ifdef USE_FADVISE
+  begin = lastpos = ahead = 0;
+  cachedstart = 0;
+  cachedend = 0;
+  readahead = KILOBYTE(128);
   written = 0;
+  totwritten = 0;
+  if (fd >= 0)
+     posix_fadvise(fd, 0, 0, POSIX_FADV_RANDOM); // we could use POSIX_FADV_SEQUENTIAL, but we do our own readahead, disabling the kernel one.
+#endif
   return fd;
 }
 
@@ -1082,16 +1090,9 @@ int cUnbufferedFile::Close(void)
 {
 #ifdef USE_FADVISE
   if (fd >= 0) {
-     if (ahead > end)
+     if (totwritten)    // if we wrote anything make sure the data has hit the disk before
-        end = ahead;
+        fdatasync(fd);  // calling fadvise, as this is our last chance to un-cache it.
-     if (begin >= 0 && end > begin) {
+     posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
-        //dsyslog("close buffer: %d (flush: %d bytes, %ld-%ld)", fd, written, begin, end);
-        if (written)
-           fdatasync(fd);
-        posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
-        }
-     begin = end = ahead = -1;
-     written = 0;
      }
 #endif
   int OldFd = fd;
@@ -1099,45 +1100,81 @@ int cUnbufferedFile::Close(void)
   return close(OldFd);
 }
 
+// When replaying and going e.g. FF->PLAY the position jumps back 2..8M
+// hence we do not want to drop recently accessed data at once.
+// We try to handle the common cases such as PLAY->FF->PLAY, small
+// jumps, moving editing marks etc.
+
+#define FADVGRAN   KILOBYTE(4) // AKA fadvise-chunk-size; PAGE_SIZE or getpagesize(2) would also work.
+#define READCHUNK  MEGABYTE(8)
+
+void cUnbufferedFile::SetReadAhead(size_t ra)
+{
+  readahead = ra;
+}
+
+int cUnbufferedFile::FadviseDrop(off_t Offset, off_t Len)
+{
+  // rounding up the window to make sure that not PAGE_SIZE-aligned data gets freed.
+  return posix_fadvise(fd, Offset - (FADVGRAN - 1), Len + (FADVGRAN - 1) * 2, POSIX_FADV_DONTNEED);
+}
+
 off_t cUnbufferedFile::Seek(off_t Offset, int Whence)
 {
-  if (fd >= 0)
+  if (Whence == SEEK_SET && Offset == curpos)
-     return lseek(fd, Offset, Whence);
+     return curpos;
-  return -1;
+  curpos = lseek(fd, Offset, Whence);
+  return curpos;
 }
 
 ssize_t cUnbufferedFile::Read(void *Data, size_t Size)
 {
   if (fd >= 0) {
 #ifdef USE_FADVISE
-     off_t pos = lseek(fd, 0, SEEK_CUR);
+     off_t jumped = curpos-lastpos; // nonzero means we're not at the last offset
-     // jump forward - adjust end position
+     if ((cachedstart < cachedend) && (curpos < cachedstart || curpos > cachedend)) {
-     if (pos > end)
+        FadviseDrop(cachedstart, cachedend-cachedstart);
-        end = pos;
+        cachedstart = curpos;
-     // after adjusting end - don't clear more than previously requested
+        cachedend = curpos;
-     if (end > ahead)
+        }
-        end = ahead;
+     cachedstart = min(cachedstart, curpos);
-     // jump backward - drop read ahead of previous run
-     if (pos < begin)
-        end = ahead;
-     if (begin >= 0 && end > begin)
-        posix_fadvise(fd, begin - KILOBYTE(200), end - begin + KILOBYTE(200), POSIX_FADV_DONTNEED);//XXX macros/parameters???
-     begin = pos;
 #endif
      ssize_t bytesRead = safe_read(fd, Data, Size);
 #ifdef USE_FADVISE
      if (bytesRead > 0) {
-        pos += bytesRead;
+        curpos += bytesRead;
-        end = pos;
+        cachedend = max(cachedend, curpos);
-        // this seems to trigger a non blocking read - this
+
-        // may or may not have been finished when we will be called next time.
+        // Read ahead:
-        // If it is not finished we can't release the not yet filled buffers.
+        // no jump? (allow small forward jump still inside readahead window).
-        // So this is commented out till we find a better solution.
+        if (jumped >= 0 && jumped <= (off_t)readahead) {
-        //posix_fadvise(fd, pos, READ_AHEAD, POSIX_FADV_WILLNEED);
+           // Trigger the readahead IO, but only if we've used at least
-        ahead = pos + READ_AHEAD;
+           // 1/2 of the previously requested area. This avoids calling
+           // fadvise() after every read() call.
+           if (ahead - curpos < (off_t)(readahead - readahead / 2)) {
+              posix_fadvise(fd, curpos, readahead, POSIX_FADV_WILLNEED);
+              ahead = curpos + readahead;
+              cachedend = max(cachedend, ahead);
+              }
+           if (readahead < Size * 32) { // automagically tune readahead size.
+              readahead = Size * 32;
+              }
            }
         else
-        end = pos;
+           ahead = curpos; // jumped -> we really don't want any readahead. otherwise eg fast-rewind gets in trouble.
+        }
+
+     if (cachedstart < cachedend) {
+        if (curpos - cachedstart > READCHUNK * 2) {
+           FadviseDrop(cachedstart, curpos - READCHUNK - cachedstart);
+           cachedstart = curpos - READCHUNK;
+           }
+        else if (cachedend > ahead && cachedend - curpos > READCHUNK * 2) {
+           FadviseDrop(curpos + READCHUNK, cachedend - curpos + READCHUNK);
+           cachedend = curpos + READCHUNK;
+           }
+        }
+     lastpos = curpos;
 #endif
      return bytesRead;
      }
@@ -1147,28 +1184,34 @@ ssize_t cUnbufferedFile::Read(void *Data, size_t Size)
 ssize_t cUnbufferedFile::Write(const void *Data, size_t Size)
 {
   if (fd >=0) {
-#ifdef USE_FADVISE
-     off_t pos = lseek(fd, 0, SEEK_CUR);
-#endif
      ssize_t bytesWritten = safe_write(fd, Data, Size);
 #ifdef USE_FADVISE
-     if (bytesWritten >= 0) {
+     if (bytesWritten > 0) {
+        begin = min(begin, curpos);
+        curpos += bytesWritten;
         written += bytesWritten;
-        if (begin >= 0) {
+        lastpos = max(lastpos, curpos);
-           if (pos < begin)
-              begin = pos;
-           }
-        else
-           begin = pos;
-        if (pos + bytesWritten > end)
-           end = pos + bytesWritten;
         if (written > WRITE_BUFFER) {
-           //dsyslog("flush buffer: %d (%d bytes, %ld-%ld)", fd, written, begin, end);
+           if (lastpos > begin) {
-           fdatasync(fd);
+              off_t headdrop = min(begin, WRITE_BUFFER * 2L);
-           if (begin >= 0 && end > begin)
+              posix_fadvise(fd, begin - headdrop, lastpos - begin + headdrop, POSIX_FADV_DONTNEED);
-              posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
+              }
-           begin = end = -1;
+           begin = lastpos = max(0L, curpos - (KILOBYTE(4) - 1));
+           totwritten += written;
            written = 0;
+           // The above fadvise() works when writing slowly (recording), but could
+           // leave cached data around when writing at a high rate (cutting).
+           // Also, it seems in some setups, the above does not trigger any I/O and
+           // the fdatasync() call below has to do all the work (reiserfs with some
+           // kind of write gathering enabled).
+           // We add 'readahead' to the threshold in an attempt to increase cutting
+           // speed; it's a tradeoff -- speed vs RAM-used.
+           if (totwritten > MEGABYTE(32) + readahead) {
+              //fdatasync(fd);
+              off_t headdrop = min(curpos - totwritten, totwritten * 2L);
+              posix_fadvise(fd, curpos - totwritten - headdrop, totwritten + headdrop, POSIX_FADV_DONTNEED);
+              totwritten = 0;
+              }
            }
         }
 #endif

tools.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.h 1.90 2006/01/15 16:19:56 kls Exp $
+ * $Id: tools.h 1.91 2006/02/04 13:58:01 kls Exp $
  */
 
 #ifndef __TOOLS_H
@@ -242,15 +242,22 @@ public:
 class cUnbufferedFile {
 private:
   int fd;
+  off_t curpos;
+  off_t cachedstart;
+  off_t cachedend;
   off_t begin;
-  off_t end;
+  off_t lastpos;
   off_t ahead;
-  ssize_t written;
+  size_t readahead;
+  size_t written;
+  size_t totwritten;
 public:
   cUnbufferedFile(void);
   ~cUnbufferedFile();
   int Open(const char *FileName, int Flags, mode_t Mode = DEFFILEMODE);
   int Close(void);
+  void SetReadAhead(size_t ra);
+  int FadviseDrop(off_t Offset, off_t Len);
   off_t Seek(off_t Offset, int Whence);
   ssize_t Read(void *Data, size_t Size);
   ssize_t Write(const void *Data, size_t Size);