frodovdr
/
vdr
mirror of https://github.com/VDR4Arch/vdr.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Improved cUnbufferedFile

This commit is contained in:
Klaus Schmidinger 2006-02-04 14:12:17 +01:00
parent c712027520
commit 116abbe05e
5 changed files with 121 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CONTRIBUTORS
View File

@ -271,6 +271,7 @@ Artur Skawina <skawina@geocities.com>
for improving the font file generation in the Makefile
for pointing out a problem with the ERR macro defined by ncurses.h
for a patch that contained a fix for checking toFile in cCuttingThread::Action()
for improving cUnbufferedFile
Werner Fink <werner@suse.de>
for making I/O more robust by handling EINTR

3
HISTORY
View File

@ -4293,3 +4293,6 @@ Video Disk Recorder Revision History
(reported by Udo Richter).
- Fixed handling the "Setup/OSD/Menu button closes" option when set to 'yes' in
case a replay is active (thanks to Udo Richter).
- Improved cUnbufferedFile; USE_FADVISE is now defined in tools.c by default, so
if you don't want to use "fadvise" you need to comment out that line (thanks to
Artur Skawina).

7
cutter.c
View File

@ -4,7 +4,7 @@
* See the main source file 'vdr.c' for copyright information and
* how to reach the author.
*
* $Id: cutter.c 1.12 2006/01/27 13:45:00 kls Exp $
* $Id: cutter.c 1.13 2006/02/04 13:40:20 kls Exp $
*/
#include "cutter.h"
@ -66,6 +66,8 @@ void cCuttingThread::Action(void)
toFile = toFileName->Open();
if (!fromFile || !toFile)
return;
fromFile->SetReadAhead(MEGABYTE(20));
toFile->SetReadAhead(MEGABYTE(20));
int Index = Mark->position;
Mark = fromMarks.Next(Mark);
int FileSize = 0;
@ -90,6 +92,7 @@ void cCuttingThread::Action(void)
if (fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) {
if (FileNumber != CurrentFileNumber) {
fromFile = fromFileName->SetOffset(FileNumber, FileOffset);
fromFile->SetReadAhead(MEGABYTE(20));
CurrentFileNumber = FileNumber;
}
if (fromFile) {
@ -122,6 +125,7 @@ void cCuttingThread::Action(void)
error = "toFile 1";
break;
}
toFile->SetReadAhead(MEGABYTE(20));
FileSize = 0;
}
LastIFrame = 0;
@ -162,6 +166,7 @@ void cCuttingThread::Action(void)
error = "toFile 2";
break;
}
toFile->SetReadAhead(MEGABYTE(20));
FileSize = 0;
}
}

159
tools.c
View File

@ -4,7 +4,7 @@
* See the main source file 'vdr.c' for copyright information and
* how to reach the author.
*
* $Id: tools.c 1.112 2006/01/20 14:01:28 kls Exp $
* $Id: tools.c 1.113 2006/02/04 14:07:30 kls Exp $
*/
#include "tools.h"
@ -1054,10 +1054,9 @@ bool cSafeFile::Close(void)
// --- cUnbufferedFile -------------------------------------------------------
//#define USE_FADVISE
#define USE_FADVISE
#define READ_AHEAD MEGABYTE(2)
#define WRITE_BUFFER MEGABYTE(10)
#define WRITE_BUFFER KILOBYTE(800)
cUnbufferedFile::cUnbufferedFile(void)
{
@ -1073,8 +1072,17 @@ int cUnbufferedFile::Open(const char *FileName, int Flags, mode_t Mode)
{
Close();
fd = open(FileName, Flags, Mode);
begin = end = ahead = -1;
curpos = 0;
#ifdef USE_FADVISE
begin = lastpos = ahead = 0;
cachedstart = 0;
cachedend = 0;
readahead = KILOBYTE(128);
written = 0;
totwritten = 0;
if (fd >= 0)
posix_fadvise(fd, 0, 0, POSIX_FADV_RANDOM); // we could use POSIX_FADV_SEQUENTIAL, but we do our own readahead, disabling the kernel one.
#endif
return fd;
}
@ -1082,16 +1090,9 @@ int cUnbufferedFile::Close(void)
{
#ifdef USE_FADVISE
if (fd >= 0) {
if (ahead > end)
end = ahead;
if (begin >= 0 && end > begin) {
//dsyslog("close buffer: %d (flush: %d bytes, %ld-%ld)", fd, written, begin, end);
if (written)
fdatasync(fd);
posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
}
begin = end = ahead = -1;
written = 0;
if (totwritten) // if we wrote anything make sure the data has hit the disk before
fdatasync(fd); // calling fadvise, as this is our last chance to un-cache it.
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
}
#endif
int OldFd = fd;
@ -1099,45 +1100,81 @@ int cUnbufferedFile::Close(void)
return close(OldFd);
}
// When replaying and going e.g. FF->PLAY the position jumps back 2..8M
// hence we do not want to drop recently accessed data at once.
// We try to handle the common cases such as PLAY->FF->PLAY, small
// jumps, moving editing marks etc.
#define FADVGRAN KILOBYTE(4) // AKA fadvise-chunk-size; PAGE_SIZE or getpagesize(2) would also work.
#define READCHUNK MEGABYTE(8)
void cUnbufferedFile::SetReadAhead(size_t ra)
{
readahead = ra;
}
int cUnbufferedFile::FadviseDrop(off_t Offset, off_t Len)
{
// rounding up the window to make sure that not PAGE_SIZE-aligned data gets freed.
return posix_fadvise(fd, Offset - (FADVGRAN - 1), Len + (FADVGRAN - 1) * 2, POSIX_FADV_DONTNEED);
}
off_t cUnbufferedFile::Seek(off_t Offset, int Whence)
{
if (fd >= 0)
return lseek(fd, Offset, Whence);
return -1;
if (Whence == SEEK_SET && Offset == curpos)
return curpos;
curpos = lseek(fd, Offset, Whence);
return curpos;
}
ssize_t cUnbufferedFile::Read(void *Data, size_t Size)
{
if (fd >= 0) {
#ifdef USE_FADVISE
off_t pos = lseek(fd, 0, SEEK_CUR);
// jump forward - adjust end position
if (pos > end)
end = pos;
// after adjusting end - don't clear more than previously requested
if (end > ahead)
end = ahead;
// jump backward - drop read ahead of previous run
if (pos < begin)
end = ahead;
if (begin >= 0 && end > begin)
posix_fadvise(fd, begin - KILOBYTE(200), end - begin + KILOBYTE(200), POSIX_FADV_DONTNEED);//XXX macros/parameters???
begin = pos;
off_t jumped = curpos-lastpos; // nonzero means we're not at the last offset
if ((cachedstart < cachedend) && (curpos < cachedstart || curpos > cachedend)) {
FadviseDrop(cachedstart, cachedend-cachedstart);
cachedstart = curpos;
cachedend = curpos;
}
cachedstart = min(cachedstart, curpos);
#endif
ssize_t bytesRead = safe_read(fd, Data, Size);
#ifdef USE_FADVISE
if (bytesRead > 0) {
pos += bytesRead;
end = pos;
// this seems to trigger a non blocking read - this
// may or may not have been finished when we will be called next time.
// If it is not finished we can't release the not yet filled buffers.
// So this is commented out till we find a better solution.
//posix_fadvise(fd, pos, READ_AHEAD, POSIX_FADV_WILLNEED);
ahead = pos + READ_AHEAD;
curpos += bytesRead;
cachedend = max(cachedend, curpos);
// Read ahead:
// no jump? (allow small forward jump still inside readahead window).
if (jumped >= 0 && jumped <= (off_t)readahead) {
// Trigger the readahead IO, but only if we've used at least
// 1/2 of the previously requested area. This avoids calling
// fadvise() after every read() call.
if (ahead - curpos < (off_t)(readahead - readahead / 2)) {
posix_fadvise(fd, curpos, readahead, POSIX_FADV_WILLNEED);
ahead = curpos + readahead;
cachedend = max(cachedend, ahead);
}
if (readahead < Size * 32) { // automagically tune readahead size.
readahead = Size * 32;
}
}
else
ahead = curpos; // jumped -> we really don't want any readahead. otherwise eg fast-rewind gets in trouble.
}
else
end = pos;
if (cachedstart < cachedend) {
if (curpos - cachedstart > READCHUNK * 2) {
FadviseDrop(cachedstart, curpos - READCHUNK - cachedstart);
cachedstart = curpos - READCHUNK;
}
else if (cachedend > ahead && cachedend - curpos > READCHUNK * 2) {
FadviseDrop(curpos + READCHUNK, cachedend - curpos + READCHUNK);
cachedend = curpos + READCHUNK;
}
}
lastpos = curpos;
#endif
return bytesRead;
}
@ -1147,28 +1184,34 @@ ssize_t cUnbufferedFile::Read(void *Data, size_t Size)
ssize_t cUnbufferedFile::Write(const void *Data, size_t Size)
{
if (fd >=0) {
#ifdef USE_FADVISE
off_t pos = lseek(fd, 0, SEEK_CUR);
#endif
ssize_t bytesWritten = safe_write(fd, Data, Size);
#ifdef USE_FADVISE
if (bytesWritten >= 0) {
if (bytesWritten > 0) {
begin = min(begin, curpos);
curpos += bytesWritten;
written += bytesWritten;
if (begin >= 0) {
if (pos < begin)
begin = pos;
}
else
begin = pos;
if (pos + bytesWritten > end)
end = pos + bytesWritten;
lastpos = max(lastpos, curpos);
if (written > WRITE_BUFFER) {
//dsyslog("flush buffer: %d (%d bytes, %ld-%ld)", fd, written, begin, end);
fdatasync(fd);
if (begin >= 0 && end > begin)
posix_fadvise(fd, begin, end - begin, POSIX_FADV_DONTNEED);
begin = end = -1;
if (lastpos > begin) {
off_t headdrop = min(begin, WRITE_BUFFER * 2L);
posix_fadvise(fd, begin - headdrop, lastpos - begin + headdrop, POSIX_FADV_DONTNEED);
}
begin = lastpos = max(0L, curpos - (KILOBYTE(4) - 1));
totwritten += written;
written = 0;
// The above fadvise() works when writing slowly (recording), but could
// leave cached data around when writing at a high rate (cutting).
// Also, it seems in some setups, the above does not trigger any I/O and
// the fdatasync() call below has to do all the work (reiserfs with some
// kind of write gathering enabled).
// We add 'readahead' to the threshold in an attempt to increase cutting
// speed; it's a tradeoff -- speed vs RAM-used.
if (totwritten > MEGABYTE(32) + readahead) {
//fdatasync(fd);
off_t headdrop = min(curpos - totwritten, totwritten * 2L);
posix_fadvise(fd, curpos - totwritten - headdrop, totwritten + headdrop, POSIX_FADV_DONTNEED);
totwritten = 0;
}
}
}
#endif

13
tools.h
View File

@ -4,7 +4,7 @@
* See the main source file 'vdr.c' for copyright information and
* how to reach the author.
*
* $Id: tools.h 1.90 2006/01/15 16:19:56 kls Exp $
* $Id: tools.h 1.91 2006/02/04 13:58:01 kls Exp $
*/
#ifndef __TOOLS_H
@ -242,15 +242,22 @@ public:
class cUnbufferedFile {
private:
int fd;
off_t curpos;
off_t cachedstart;
off_t cachedend;
off_t begin;
off_t end;
off_t lastpos;
off_t ahead;
ssize_t written;
size_t readahead;
size_t written;
size_t totwritten;
public:
cUnbufferedFile(void);
~cUnbufferedFile();
int Open(const char *FileName, int Flags, mode_t Mode = DEFFILEMODE);
int Close(void);
void SetReadAhead(size_t ra);
int FadviseDrop(off_t Offset, off_t Len);
off_t Seek(off_t Offset, int Whence);
ssize_t Read(void *Data, size_t Size);
ssize_t Write(const void *Data, size_t Size);