The new class cUnbufferedFile is used for the recording files to avoid thrashing the file system cache

CONTRIBUTORS

@@ -1520,3 +1520,6 @@ Nicolas Huillard <nhuillard@e-dition.fr>
 
 Patrick Fischer <patrick_fischer@gmx.de>
  for reporting an error in the cFilter example in PLUGINS.html
+
+Ralf M<>ller <ralf@bj-ig.de>
+ for a patch that was used to implement cUnbufferedFile

HISTORY

@@ -3886,7 +3886,7 @@ Video Disk Recorder Revision History
 - The 'sub-title' and 'bottom text' in the CAM menu can now consist of several lines.
 - Improved the CAM enquiry menu.
 
-2005-10-30: Version 1.3.35
+2005-10-31: Version 1.3.35
 
 - Updated 'sources.conf' (thanks to Philip Prindeville).
 - Now using daemon() instead of fork() to run VDR in daemon mode (thanks to
@@ -3910,3 +3910,5 @@ Video Disk Recorder Revision History
 - Updated the Greek OSD texts (thanks to Dimitrios Dimitrakos).
 - Updated the French OSD texts (thanks to Nicolas Huillard).
 - Fixed the cFilter example in PLUGINS.html (reported by Patrick Fischer).
+- The new class cUnbufferedFile is used for the recording files to avoid
+  trashing the file system cache (based on a patch by Ralf M<>ller).

cutter.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: cutter.c 1.10 2005/08/14 10:51:54 kls Exp $
+ * $Id: cutter.c 1.11 2005/10/31 12:26:44 kls Exp $
  */
 
 #include "cutter.h"
@@ -18,7 +18,7 @@
 class cCuttingThread : public cThread {
 private:
   const char *error;
-  int fromFile, toFile;
+  cUnbufferedFile *fromFile, *toFile;
   cFileName *fromFileName, *toFileName;
   cIndexFile *fromIndex, *toIndex;
   cMarks fromMarks, toMarks;
@@ -34,7 +34,7 @@ cCuttingThread::cCuttingThread(const char *FromFileName, const char *ToFileName)
 :cThread("video cutting")
 {
   error = NULL;
-  fromFile = toFile = -1;
+  fromFile = toFile = NULL;
   fromFileName = toFileName = NULL;
   fromIndex = toIndex = NULL;
   if (fromMarks.Load(FromFileName) && fromMarks.Count()) {
@@ -64,7 +64,7 @@ void cCuttingThread::Action(void)
   if (Mark) {
      fromFile = fromFileName->Open();
      toFile = toFileName->Open();
-     if (fromFile < 0 || toFile < 0)
+     if (!fromFile || !toFile)
         return;
      int Index = Mark->position;
      Mark = fromMarks.Next(Mark);
@@ -92,7 +92,7 @@ void cCuttingThread::Action(void)
                  fromFile = fromFileName->SetOffset(FileNumber, FileOffset);
                  CurrentFileNumber = FileNumber;
                  }
-              if (fromFile >= 0) {
+              if (fromFile) {
                  int len = ReadFrame(fromFile, buffer,  Length, sizeof(buffer));
                  if (len < 0) {
                     error = "ReadFrame";
@@ -131,7 +131,7 @@ void cCuttingThread::Action(void)
                  cutIn = false;
                  }
               }
-           if (safe_write(toFile, buffer, Length) < 0) {
+           if (toFile->Write(buffer, Length) < 0) {
               error = "safe_write";
               break;
               }

dvbplayer.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: dvbplayer.c 1.40 2005/08/29 15:43:30 kls Exp $
+ * $Id: dvbplayer.c 1.41 2005/10/31 12:33:48 kls Exp $
  */
 
 #include "dvbplayer.h"
@@ -74,7 +74,7 @@ int cBackTrace::Get(bool Forward)
 
 class cNonBlockingFileReader : public cThread {
 private:
-  int f;
+  cUnbufferedFile *f;
   uchar *buffer;
   int wanted;
   int length;
@@ -86,14 +86,14 @@ public:
   cNonBlockingFileReader(void);
   ~cNonBlockingFileReader();
   void Clear(void);
-  int Read(int FileHandle, uchar *Buffer, int Length);
+  int Read(cUnbufferedFile *File, uchar *Buffer, int Length);
   bool Reading(void) { return buffer; }
   };
 
 cNonBlockingFileReader::cNonBlockingFileReader(void)
 :cThread("non blocking file reader")
 {
-  f = -1;
+  f = NULL;
   buffer = NULL;
   wanted = length = 0;
   hasData = false;
@@ -110,7 +110,7 @@ cNonBlockingFileReader::~cNonBlockingFileReader()
 void cNonBlockingFileReader::Clear(void)
 {
   Lock();
-  f = -1;
+  f = NULL;
   free(buffer);
   buffer = NULL;
   wanted = length = 0;
@@ -119,7 +119,7 @@ void cNonBlockingFileReader::Clear(void)
   newSet.Signal();
 }
 
-int cNonBlockingFileReader::Read(int FileHandle, uchar *Buffer, int Length)
+int cNonBlockingFileReader::Read(cUnbufferedFile *File, uchar *Buffer, int Length)
 {
   if (hasData && buffer) {
      if (buffer != Buffer) {
@@ -131,7 +131,7 @@ int cNonBlockingFileReader::Read(int FileHandle, uchar *Buffer, int Length)
      return length;
      }
   if (!buffer) {
-     f = FileHandle;
+     f = File;
      buffer = Buffer;
      wanted = Length;
      length = 0;
@@ -146,8 +146,8 @@ void cNonBlockingFileReader::Action(void)
 {
   while (Running()) {
         Lock();
-        if (!hasData && f >= 0 && buffer) {
+        if (!hasData && f && buffer) {
-           int r = safe_read(f, buffer + length, wanted - length);
+           int r = f->Read(buffer + length, wanted - length);
            if (r >= 0) {
               length += r;
               if (!r || length == wanted) // r == 0 means EOF
@@ -181,7 +181,7 @@ private:
   cBackTrace *backTrace;
   cFileName *fileName;
   cIndexFile *index;
-  int replayFile;
+  cUnbufferedFile *replayFile;
   bool eof;
   bool firstPacket;
   ePlayModes playMode;
@@ -237,7 +237,7 @@ cDvbPlayer::cDvbPlayer(const char *FileName)
   isyslog("replay %s", FileName);
   fileName = new cFileName(FileName, false);
   replayFile = fileName->Open();
-  if (replayFile < 0)
+  if (!replayFile)
      return;
   ringBuffer = new cRingBufferFrame(PLAYERBUFSIZE);
   // Create the index file:
@@ -302,10 +302,10 @@ bool cDvbPlayer::NextFile(uchar FileNumber, int FileOffset)
 {
   if (FileNumber > 0)
      replayFile = fileName->SetOffset(FileNumber, FileOffset);
-  else if (replayFile >= 0 && eof)
+  else if (replayFile && eof)
      replayFile = fileName->NextFile();
   eof = false;
-  return replayFile >= 0;
+  return replayFile != NULL;
 }
 
 int cDvbPlayer::Resume(void)
@@ -342,7 +342,7 @@ bool cDvbPlayer::Save(void)
 void cDvbPlayer::Activate(bool On)
 {
   if (On) {
-     if (replayFile >= 0)
+     if (replayFile)
         Start();
      }
   else
@@ -376,7 +376,7 @@ void cDvbPlayer::Action(void)
            // Read the next frame from the file:
 
            if (playMode != pmStill && playMode != pmPause) {
-              if (!readFrame && (replayFile >= 0 || readIndex >= 0)) {
+              if (!readFrame && (replayFile || readIndex >= 0)) {
                  if (!nonBlockingFileReader->Reading()) {
                     if (playMode == pmFast || (playMode == pmSlow && playDir == pdBackward)) {
                        uchar FileNumber;

recorder.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: recorder.c 1.15 2005/08/14 10:53:28 kls Exp $
+ * $Id: recorder.c 1.16 2005/10/31 12:35:29 kls Exp $
  */
 
 #include <stdarg.h>
@@ -28,7 +28,7 @@ private:
   cIndexFile *index;
   uchar pictureType;
   int fileSize;
-  int recordFile;
+  cUnbufferedFile *recordFile;
   time_t lastDiskSpaceCheck;
   bool RunningLowOnDiskSpace(void);
   bool NextFile(void);
@@ -50,7 +50,7 @@ cFileWriter::cFileWriter(const char *FileName, cRemux *Remux)
   lastDiskSpaceCheck = time(NULL);
   fileName = new cFileName(FileName, true);
   recordFile = fileName->Open();
-  if (recordFile < 0)
+  if (!recordFile)
      return;
   // Create the index file:
   index = new cIndexFile(FileName, true);
@@ -81,13 +81,13 @@ bool cFileWriter::RunningLowOnDiskSpace(void)
 
 bool cFileWriter::NextFile(void)
 {
-  if (recordFile >= 0 && pictureType == I_FRAME) { // every file shall start with an I_FRAME
+  if (recordFile && pictureType == I_FRAME) { // every file shall start with an I_FRAME
      if (fileSize > MEGABYTE(Setup.MaxVideoFileSize) || RunningLowOnDiskSpace()) {
         recordFile = fileName->NextFile();
         fileSize = 0;
         }
      }
-  return recordFile >= 0;
+  return recordFile != NULL;
 }
 
 void cFileWriter::Action(void)
@@ -102,7 +102,7 @@ void cFileWriter::Action(void)
            if (NextFile()) {
               if (index && pictureType != NO_PICTURE)
                  index->Write(pictureType, fileName->Number(), fileSize);
-              if (safe_write(recordFile, p, Count) < 0) {
+              if (recordFile->Write(p, Count) < 0) {
                  LOG_ERROR_STR(fileName->Name());
                  break;
                  }

recording.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: recording.c 1.121 2005/10/09 13:09:51 kls Exp $
+ * $Id: recording.c 1.122 2005/10/31 12:27:58 kls Exp $
  */
 
 #include "recording.h"
@@ -1258,7 +1258,7 @@ int cIndexFile::Get(uchar FileNumber, int FileOffset)
 
 cFileName::cFileName(const char *FileName, bool Record, bool Blocking)
 {
-  file = -1;
+  file = NULL;
   fileNumber = 0;
   record = Record;
   blocking = Blocking;
@@ -1279,21 +1279,21 @@ cFileName::~cFileName()
   free(fileName);
 }
 
-int cFileName::Open(void)
+cUnbufferedFile *cFileName::Open(void)
 {
-  if (file < 0) {
+  if (!file) {
      int BlockingFlag = blocking ? 0 : O_NONBLOCK;
      if (record) {
         dsyslog("recording to '%s'", fileName);
         file = OpenVideoFile(fileName, O_RDWR | O_CREAT | BlockingFlag);
-        if (file < 0)
+        if (!file)
            LOG_ERROR_STR(fileName);
         }
      else {
         if (access(fileName, R_OK) == 0) {
            dsyslog("playing '%s'", fileName);
-           file = open(fileName, O_RDONLY | BlockingFlag);
+           file = cUnbufferedFile::Create(fileName, O_RDONLY | BlockingFlag);
-           if (file < 0)
+           if (!file)
               LOG_ERROR_STR(fileName);
            }
         else if (errno != ENOENT)
@@ -1305,14 +1305,14 @@ int cFileName::Open(void)
 
 void cFileName::Close(void)
 {
-  if (file >= 0) {
+  if (file) {
-     if ((record && CloseVideoFile(file) < 0) || (!record && close(file) < 0))
+     if ((record && CloseVideoFile(file) < 0) || (!record && file->Close() < 0))
         LOG_ERROR_STR(fileName);
-     file = -1;
+     file = NULL;
      }
 }
 
-int cFileName::SetOffset(int Number, int Offset)
+cUnbufferedFile *cFileName::SetOffset(int Number, int Offset)
 {
   if (fileNumber != Number)
      Close();
@@ -1337,23 +1337,23 @@ int cFileName::SetOffset(int Number, int Offset)
            }
         else if (errno != ENOENT) { // something serious has happened
            LOG_ERROR_STR(fileName);
-           return -1;
+           return NULL;
            }
         // found a non existing file suffix
         }
      if (Open() >= 0) {
-        if (!record && Offset >= 0 && lseek(file, Offset, SEEK_SET) != Offset) {
+        if (!record && Offset >= 0 && file->Seek(Offset, SEEK_SET) != Offset) {
            LOG_ERROR_STR(fileName);
-           return -1;
+           return NULL;
            }
         }
      return file;
      }
   esyslog("ERROR: max number of files (%d) exceeded", MAXFILESPERRECORDING);
-  return -1;
+  return NULL;
 }
 
-int cFileName::NextFile(void)
+cUnbufferedFile *cFileName::NextFile(void)
 {
   return SetOffset(fileNumber + 1);
 }
@@ -1387,7 +1387,7 @@ int SecondsToFrames(int Seconds)
 
 // --- ReadFrame -------------------------------------------------------------
 
-int ReadFrame(int f, uchar *b, int Length, int Max)
+int ReadFrame(cUnbufferedFile *f, uchar *b, int Length, int Max)
 {
   if (Length == -1)
      Length = Max; // this means we read up to EOF (see cIndex)
@@ -1395,10 +1395,8 @@ int ReadFrame(int f, uchar *b, int Length, int Max)
      esyslog("ERROR: frame larger than buffer (%d > %d)", Length, Max);
      Length = Max;
      }
-  int r = safe_read(f, b, Length);
+  int r = f->Read(b, Length);
   if (r < 0)
      LOG_ERROR;
   return r;
 }
-
-

recording.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: recording.h 1.45 2005/10/01 10:24:41 kls Exp $
+ * $Id: recording.h 1.46 2005/10/31 12:27:12 kls Exp $
  */
 
 #ifndef __RECORDING_H
@@ -203,7 +203,7 @@ public:
 
 class cFileName {
 private:
-  int file;
+  cUnbufferedFile *file;
   int fileNumber;
   char *fileName, *pFileNumber;
   bool record;
@@ -213,10 +213,10 @@ public:
   ~cFileName();
   const char *Name(void) { return fileName; }
   int Number(void) { return fileNumber; }
-  int Open(void);
+  cUnbufferedFile *Open(void);
   void Close(void);
-  int SetOffset(int Number, int Offset = 0);
+  cUnbufferedFile *SetOffset(int Number, int Offset = 0);
-  int NextFile(void);
+  cUnbufferedFile *NextFile(void);
   };
 
 cString IndexToHMSF(int Index, bool WithFrame = false);
@@ -226,7 +226,7 @@ int HMSFToIndex(const char *HMSF);
 int SecondsToFrames(int Seconds); //XXX+ ->player???
       // Returns the number of frames corresponding to the given number of seconds.
 
-int ReadFrame(int f, uchar *b, int Length, int Max);
+int ReadFrame(cUnbufferedFile *f, uchar *b, int Length, int Max);
 
 char *ExchangeChars(char *s, bool ToFileSystem);
       // Exchanges the characters in the given string to or from a file system

tools.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.c 1.99 2005/09/25 12:56:06 kls Exp $
+ * $Id: tools.c 1.100 2005/10/31 12:56:15 kls Exp $
  */
 
 #include "tools.h"
@@ -836,6 +836,129 @@ bool cSafeFile::Close(void)
   return result;
 }
 
+// --- cUnbufferedFile -------------------------------------------------------
+
+#define READ_AHEAD MEGABYTE(2)
+#define WRITE_BUFFER MEGABYTE(10)
+
+cUnbufferedFile::cUnbufferedFile(void)
+{
+  fd = -1;
+}
+
+cUnbufferedFile::~cUnbufferedFile()
+{
+  Close();
+}
+
+int cUnbufferedFile::Open(const char *FileName, int Flags, mode_t Mode)
+{
+  Close();
+  fd = open(FileName, Flags, Mode);
+  begin = end = ahead = -1;
+  written = 0;
+  return fd;
+}
+
+int cUnbufferedFile::Close(void)
+{
+  if (fd >= 0) {
+     if (ahead > end)
+        end = ahead;
+     if (begin >= 0 && end > begin) {
+        //dsyslog("close buffer: %d (flush: %d bytes, %ld-%ld)", fd, written, begin, end);
+        if (written)
+           fdatasync(fd);
+        posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
+        }
+     begin = end = ahead = -1;
+     written = 0;
+     }
+  int OldFd = fd;
+  fd = -1;
+  return close(OldFd);
+}
+
+off_t cUnbufferedFile::Seek(off_t Offset, int Whence)
+{
+  if (fd >= 0)
+     return lseek(fd, Offset, Whence);
+  return -1;
+}
+
+ssize_t cUnbufferedFile::Read(void *Data, size_t Size)
+{
+  if (fd >= 0) {
+     off_t pos = lseek(fd, 0, SEEK_CUR);
+     // jump forward - adjust end position
+     if (pos > end)
+        end = pos;
+     // after adjusting end - don't clear more than previously requested
+     if (end > ahead)
+        end = ahead;
+     // jump backward - drop read ahead of previous run
+     if (pos < begin)
+        end = ahead;
+     if (begin >= 0 && end > begin)
+        posix_fadvise(fd, begin - KILOBYTE(200), end - begin + KILOBYTE(200), POSIX_FADV_DONTNEED);//XXX macros/parameters???
+     begin = pos;
+     ssize_t bytesRead = safe_read(fd, Data, Size);
+     if (bytesRead > 0) {
+        pos += bytesRead;
+        end = pos;
+        // this seems to trigger a non blocking read - this
+        // may or may not have been finished when we will be called next time.
+        // If it is not finished we can't release the not yet filled buffers.
+        // So this is commented out till we find a better solution.
+        //posix_fadvise(fd, pos, READ_AHEAD, POSIX_FADV_WILLNEED);
+        ahead = pos + READ_AHEAD;
+        }
+     else
+        end = pos;
+     return bytesRead;
+     }
+  return -1;
+}
+
+ssize_t cUnbufferedFile::Write(const void *Data, size_t Size)
+{
+  if (fd >=0) {
+     off_t pos = lseek(fd, 0, SEEK_CUR);
+     ssize_t bytesWritten = safe_write(fd, Data, Size);
+     if (bytesWritten >= 0) {
+        written += bytesWritten;
+        if (begin >= 0) {
+           if (pos < begin)
+              begin = pos;
+           }
+        else
+           begin = pos;
+        if (pos + bytesWritten > end)
+           end = pos + bytesWritten;
+        if (written > WRITE_BUFFER) {
+           //dsyslog("flush buffer: %d (%d bytes, %ld-%ld)", fd, written, begin, end);
+           fdatasync(fd);
+           if (begin >= 0 && end > begin)
+              posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
+           begin = end = -1;
+           written = 0;
+           }
+        }
+     return bytesWritten;
+     }
+  return -1;
+}
+
+cUnbufferedFile *cUnbufferedFile::Create(const char *FileName, int Flags, mode_t Mode)
+{
+  cUnbufferedFile *File = new cUnbufferedFile;
+  if (File->Open(FileName, Flags, Mode) < 0) {
+     delete File;
+     File = NULL;
+     }
+  return File;
+}
+
 // --- cLockFile -------------------------------------------------------------
 
 #define LOCKFILENAME      ".lock-vdr"

tools.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: tools.h 1.80 2005/10/09 11:13:06 kls Exp $
+ * $Id: tools.h 1.81 2005/10/31 12:54:36 kls Exp $
  */
 
 #ifndef __TOOLS_H
@@ -198,6 +198,27 @@ public:
   bool Close(void);
   };
 
+/// cUnbufferedFile is used for large files that are mainly written or read
+/// in a streaming manner, and thus should not be cached.
+
+class cUnbufferedFile {
+private:
+  int fd;
+  off_t begin;
+  off_t end;
+  off_t ahead;
+  ssize_t written;
+public:
+  cUnbufferedFile(void);
+  ~cUnbufferedFile();
+  int Open(const char *FileName, int Flags, mode_t Mode = DEFFILEMODE);
+  int Close(void);
+  off_t Seek(off_t Offset, int Whence);
+  ssize_t Read(void *Data, size_t Size);
+  ssize_t Write(const void *Data, size_t Size);
+  static cUnbufferedFile *Create(const char *FileName, int Flags, mode_t Mode = DEFFILEMODE);
+  };
+
 class cLockFile {
 private:
   char *fileName;

videodir.c

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: videodir.c 1.12 2005/08/06 09:53:21 kls Exp $
+ * $Id: videodir.c 1.13 2005/10/31 12:07:41 kls Exp $
  */
 
 #include "videodir.h"
@@ -102,7 +102,7 @@ const char *cVideoDirectory::Adjust(const char *FileName)
   return NULL;
 }
 
-int OpenVideoFile(const char *FileName, int Flags)
+cUnbufferedFile *OpenVideoFile(const char *FileName, int Flags)
 {
   const char *ActualFileName = FileName;
 
@@ -110,7 +110,7 @@ int OpenVideoFile(const char *FileName, int Flags)
   if (strstr(FileName, VideoDirectory) != FileName) {
      esyslog("ERROR: %s not in %s", FileName, VideoDirectory);
      errno = ENOENT; // must set 'errno' - any ideas for a better value?
-     return -1;
+     return NULL;
      }
   // Are we going to create a new file?
   if ((Flags & O_CREAT) != 0) {
@@ -128,25 +128,26 @@ int OpenVideoFile(const char *FileName, int Flags)
         if (Dir.Stored()) {
            ActualFileName = Dir.Adjust(FileName);
            if (!MakeDirs(ActualFileName, false))
-              return -1; // errno has been set by MakeDirs()
+              return NULL; // errno has been set by MakeDirs()
            if (symlink(ActualFileName, FileName) < 0) {
               LOG_ERROR_STR(FileName);
-              return -1;
+              return NULL;
               }
            ActualFileName = strdup(ActualFileName); // must survive Dir!
            }
         }
      }
-  int Result = open(ActualFileName, Flags, DEFFILEMODE);
+  cUnbufferedFile *File = cUnbufferedFile::Create(ActualFileName, Flags, DEFFILEMODE);
   if (ActualFileName != FileName)
      free((char *)ActualFileName);
-  return Result;
+  return File;
 }
 
-int CloseVideoFile(int FileHandle)
+int CloseVideoFile(cUnbufferedFile *File)
 {
-  // just in case we ever decide to do something special when closing the file!
+  int Result = File->Close();
-  return close(FileHandle);
+  delete File;
+  return Result;
 }
 
 bool RenameVideoFile(const char *OldName, const char *NewName)

videodir.h

@@ -4,7 +4,7 @@
  * See the main source file 'vdr.c' for copyright information and
  * how to reach the author.
  *
- * $Id: videodir.h 1.5 2004/12/26 11:52:56 kls Exp $
+ * $Id: videodir.h 1.6 2005/10/31 11:50:23 kls Exp $
  */
 
 #ifndef __VIDEODIR_H
@@ -15,8 +15,8 @@
 
 extern const char *VideoDirectory;
 
-int OpenVideoFile(const char *FileName, int Flags);
+cUnbufferedFile *OpenVideoFile(const char *FileName, int Flags);
-int CloseVideoFile(int FileHandle);
+int CloseVideoFile(cUnbufferedFile *File);
 bool RenameVideoFile(const char *OldName, const char *NewName);
 bool RemoveVideoFile(const char *FileName);
 bool VideoFileSpaceAvailable(int SizeMB);