frodovdr
/
vdr-plugin-scraper2vdr
mirror of https://projects.vdr-developer.org/git/vdr-plugin-scraper2vdr.git
1
0
Fork 0
Code Issues Releases Wiki Activity

some performance optimizations

This commit is contained in:
louis 2014-04-18 17:08:11 +02:00
parent d454507416
commit 4f90ed5cc8
5 changed files with 186 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/db.c
View File

@ -151,6 +151,11 @@ int cDbStatement::bind(int field, int mode, const char* delim)
return bind(table->getRow()->getValue(field), mode, delim);
}
int cDbStatement::bind(cDbTable* aTable, int field, int mode, const char* delim)
{
return bind(aTable->getRow()->getValue(field), mode, delim);
}
int cDbStatement::bind(cDbValue* value, int mode, const char* delim)
{
if (!value || !value->getField())
@ -185,7 +190,7 @@ int cDbStatement::bindCmp(const char* ctable, cDbValue* value,
if (ctable)
build("%s.", ctable);
build("%s%s %s ?", delim ? delim : "", value->getName(), comp);
build("%s%s%s %s ?", delim ? delim : "", bindPrefix ? bindPrefix : "", value->getName(), comp);
appendBinding(value, bndIn);
@ -201,7 +206,7 @@ int cDbStatement::bindCmp(const char* ctable, int field, cDbValue* value,
if (ctable)
build("%s.", ctable);
build("%s%s %s ?", delim ? delim : "", vf->getName(), comp);
build("%s%s%s %s ?", delim ? delim : "", bindPrefix ? bindPrefix : "", vf->getName(), comp);
appendBinding(vv, bndIn);
@ -349,7 +354,7 @@ int cDbStatement::prepare()
return connection->errorSql(connection, "buildPrimarySelect(bind_param)", stmt);
}
tell(2, "Statement '%s' with (%d) in parameters and (%d) out bindings prepared",
tell(2, "Statement '%s' with (%ld) in parameters and (%d) out bindings prepared",
stmtTxt.c_str(), mysql_stmt_param_count(stmt), outCount);
return success;
@ -457,6 +462,10 @@ int cDbTable::init()
if (createTable() != success)
return fail;
// check/create indices
createIndices();
// ------------------------------
// prepare BASIC statements
// ------------------------------
@ -676,10 +685,6 @@ int cDbTable::createTable()
return connection->errorSql(getConnection(), "createTable()",
0, statement.c_str());
// create indices
createIndices();
return success;
}

32
lib/db.h
View File

@ -12,6 +12,7 @@
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <errno.h>
#include <sstream>
@ -306,28 +307,6 @@ class cDbValue : public cDbService
return no;
}
// int hasValue(float value)
// {
// if (field->format != ffFloat)
// {
// tell(0, "Checking invalid field format for '%s', expected FLOAT", field->name);
// return no;
// }
// return floatValue == value;
// }
// int hasValue(long value)
// {
// if (field->format != ffInt && field->format != ffUInt)
// {
// tell(0, "Checking invalid field format for '%s', expected INT", field->name);
// return no;
// }
// return numValue == value;
// }
int hasValue(const char* value)
{
if (!value)
@ -345,8 +324,9 @@ class cDbValue : public cDbService
time_t getTimeValue()
{
struct tm tm;
memset(&tm, 0, sizeof(tm));
tm.tm_isdst = -1; // force DST auto detect
tm.tm_year = timeValue.year - 1900;
tm.tm_mon = timeValue.month - 1;
tm.tm_mday = timeValue.day;
@ -409,6 +389,7 @@ class cDbStatement : public cDbService
void setBindPrefix(const char* p) { bindPrefix = p; }
void clrBindPrefix() { bindPrefix = 0; }
int bind(cDbValue* value, int mode, const char* delim = 0);
int bind(cDbTable* aTable, int field, int mode, const char* delim);
int bind(int field, int mode, const char* delim = 0);
int bindCmp(const char* table, cDbValue* value,
@ -634,10 +615,10 @@ class cDbConnection
if (!getMySql())
return fail;
if (!(f = fopen(file, "r")))
{
tell(0, "Fatal: Can't access '%s'; %m", file);
tell(0, "Fatal: Can't access '%s'; %s", file, strerror(errno));
return fail;
}
@ -816,6 +797,7 @@ class cDbTable : public cDbService
int isNull(int f) const { return row->isNull(f); }
FieldDef* getField(int f) { return row->getField(f); }
cDbValue* getValue(int f) { return row->getValue(f); }
int fieldCount() { return row->fieldCount(); }
cDbRow* getRow() { return row; }

120
lib/tabledef.c
View File

@ -146,14 +146,15 @@ cDbService::IndexDef cTableEvents::indices[] =
{
// index fields
{ "comptitle", { fiCompTitle, na }, 0 },
{ "source", { fiSource, na }, 0 },
{ "FilerefSource", { fiFileRef, fiSource, na }, 0 },
{ "channelid", { fiChannelId, na }, 0 },
{ "useid", { fiUseId, na }, 0 },
{ "useidchannelid", { fiUseId, fiChannelId, na }, 0 },
{ "updflgupdsp", { fiUpdFlg, fiUpdSp, na }, 0 },
{ "idxsourcechannelid", { fiSource, fiChannelId, na }, 0 },
{ "comptitle", { fiCompTitle, na }, 0 },
{ "source", { fiSource, na }, 0 },
{ "FilerefSource", { fiFileRef, fiSource, na }, 0 },
{ "channelid", { fiChannelId, na }, 0 },
{ "useid", { fiUseId, na }, 0 },
{ "useidchannelid", { fiUseId, fiChannelId, na }, 0 },
{ "updflgupdsp", { fiUpdFlg, fiUpdSp, na }, 0 },
{ "sourcechannelid", { fiSource, fiChannelId, na }, 0 },
{ "scrsp", { fiScrSp, na }, 0 },
{ 0 }
};
@ -335,12 +336,14 @@ cDbService::FieldDef cTableChannelMap::fields[] =
cDbService::IndexDef cTableChannelMap::indices[] =
{
// index fields
// index fields
{ "sourceExtid", { fiSource, fiExternalId, na }, 0 },
{ "source", { fiSource, na }, 0 },
{ "updflg", { fiUpdFlg, na }, 0 },
{ "idxsourcechannelid", { fiSource, fiChannelId, na }, 0 },
{ "sourcechannelid", { fiSource, fiChannelId, na }, 0 },
{ "mergesp", { fiMergeSp, na }, 0 },
{ "channelid", { fiChannelId, na }, 0 },
{ 0 }
};
@ -370,6 +373,15 @@ cDbService::FieldDef cTableVdrs::fields[] =
{ 0 }
};
cDbService::IndexDef cTableVdrs::indices[] =
{
// index fields
{ "state", { fiState, na }, 0 },
{ 0 }
};
//***************************************************************************
// Parameter Fields
//***************************************************************************
@ -472,6 +484,27 @@ cDbService::IndexDef cTableSnapshot::indices[] =
{ 0 }
};
//***************************************************************************
// Timers
//***************************************************************************
cDbService::FieldDef cTableTimers::fields[] =
{
// name format size index type
{ "eventid", ffUInt, 0, fiEventId, ftPrimary },
{ "channelid", ffAscii, 50, fiChannelId, ftPrimary },
{ "vdruuid", ffAscii, 40, fiVdrUuid, ftPrimary },
{ "inssp", ffInt, 0, fiInsSp, ftMeta },
{ "updsp", ffInt, 0, fiUpdSp, ftMeta },
{ "state", ffAscii, 1, fiState, ftData }, // { 'D'eleted, 'N'ew, 'A'ssigned, pease 'R'eassign }
{ "starttime", ffInt, 10, fiStartTime, ftData },
{ "endtime", ffInt, 10, fiEndTime, ftData },
{ 0 }
};
//***************************************************************************
// Series Fields
@ -482,8 +515,11 @@ cDbService::FieldDef cTableSeries::fields[] =
// name format size index type
// primary key
{ "series_id", ffUInt, 0, fiSeriesId, ftPrimary },
//Data
// data
{ "series_name", ffAscii, 200, fiSeriesName, ftData },
{ "series_last_scraped", ffUInt, 0, fiSeriesLastScraped, ftData },
{ "series_last_updated", ffUInt, 0, fiSeriesLastUpdated, ftData },
@ -511,8 +547,10 @@ cDbService::FieldDef* cTableSeries::toField(const char* name)
cDbService::IndexDef cTableSeries::indices[] =
{
// index fields
// index fields
{ "seriesname", { fiSeriesName, na }, 0 },
{ 0 }
};
@ -526,8 +564,11 @@ cDbService::FieldDef cTableSeriesEpisode::fields[] =
// name format size index type
// primary key
{ "episode_id", ffUInt, 0, fiEpisodeId, ftPrimary },
//Data
// data
{ "episode_number", ffUInt, 0, fiEpisodeNumber, ftData },
{ "season_number", ffUInt, 0, fiSeasonNumber, ftData },
{ "episode_name", ffAscii, 300, fiEpisodeName, ftData },
@ -569,12 +610,15 @@ cDbService::FieldDef cTableSeriesMedia::fields[] =
// name format size index type
// primary key
{ "series_id", ffUInt, 0, fiSeriesId, ftPrimary },
{ "season_number", ffUInt, 0, fiSeasonNumber, ftPrimary },
{ "episode_id", ffUInt, 0, fiEpisodeId, ftPrimary },
{ "actor_id", ffUInt, 0, fiActorId, ftPrimary },
{ "media_type", ffUInt, 0, fiMediaType, ftPrimary },
//Data
// data
{ "media_url", ffAscii, 100, fiMediaUrl, ftData },
{ "media_width", ffUInt, 0, fiMediaWidth, ftData },
{ "media_height", ffUInt, 0, fiMediaHeight, ftData },
@ -598,6 +642,7 @@ cDbService::FieldDef* cTableSeriesMedia::toField(const char* name)
cDbService::IndexDef cTableSeriesMedia::indices[] =
{
// index fields
{ "series_id", { fiSeriesId, na }, 0 },
{ "season_number", { fiSeasonNumber, na }, 0 },
{ "episode_id", { fiEpisodeId, na }, 0 },
@ -615,8 +660,11 @@ cDbService::FieldDef cTableSeriesActor::fields[] =
// name format size index type
// primary key
{ "actor_id", ffUInt, 0, fiActorId, ftPrimary },
//Data
// data
{ "actor_name", ffAscii, 100, fiActorName, ftData },
{ "actor_role", ffAscii, 500, fiActorRole, ftData },
{ "actor_sortorder", ffUInt, 0, fiSortOrder, ftData },
@ -651,8 +699,11 @@ cDbService::FieldDef cTableMovies::fields[] =
// name format size index type
// primary key
{ "movie_id", ffUInt, 0, fiMovieId, ftPrimary },
//Data
// data
{ "movie_title", ffAscii, 300, fiTitle, ftData },
{ "movie_original_title", ffAscii, 300, fiOriginalTitle, ftData },
{ "movie_tagline", ffAscii, 1000, fiTagline, ftData },
@ -685,8 +736,10 @@ cDbService::FieldDef* cTableMovies::toField(const char* name)
cDbService::IndexDef cTableMovies::indices[] =
{
// index fields
// index fields
{ "movie_id", { fiMovieId, na }, 0 },
{ "movietitle", { fiTitle, na }, 0 },
{ 0 }
};
@ -700,8 +753,11 @@ cDbService::FieldDef cTableMovieActor::fields[] =
// name format size index type
// primary key
{ "actor_id", ffUInt, 0, fiActorId, ftPrimary },
//Data
// data
{ "actor_name", ffAscii, 300, fiActorName, ftData },
{ 0 }
@ -721,6 +777,7 @@ cDbService::FieldDef* cTableMovieActor::toField(const char* name)
cDbService::IndexDef cTableMovieActor::indices[] =
{
// index fields
{ "actor_id", { fiActorId, na }, 0 },
{ 0 }
@ -735,9 +792,12 @@ cDbService::FieldDef cTableMovieActors::fields[] =
// name format size index type
// primary key
{ "movie_id", ffUInt, 0, fiMovieId, ftPrimary },
{ "actor_id", ffUInt, 0, fiActorId, ftPrimary },
//Data
// data
{ "actor_role", ffAscii, 300, fiRole, ftData },
{ 0 }
@ -757,6 +817,7 @@ cDbService::FieldDef* cTableMovieActors::toField(const char* name)
cDbService::IndexDef cTableMovieActors::indices[] =
{
// index fields
{ "movie_id", { fiMovieId, na }, 0 },
{ "actor_id", { fiActorId, na }, 0 },
@ -772,10 +833,13 @@ cDbService::FieldDef cTableMovieMedia::fields[] =
// name format size index type
// primary key
{ "movie_id", ffUInt, 0, fiMovieId, ftPrimary },
{ "actor_id", ffUInt, 0, fiActorId, ftPrimary },
{ "media_type", ffUInt, 0, fiMediaType, ftPrimary },
//Data
// data
{ "media_url", ffAscii, 100, fiMediaUrl, ftData },
{ "media_width", ffUInt, 0, fiMediaWidth, ftData },
{ "media_height", ffUInt, 0, fiMediaHeight, ftData },
@ -798,6 +862,7 @@ cDbService::FieldDef* cTableMovieMedia::toField(const char* name)
cDbService::IndexDef cTableMovieMedia::indices[] =
{
// index fields
{ "movie_id", { fiMovieId, na }, 0 },
{ "actor_id", { fiActorId, na }, 0 },
@ -813,11 +878,13 @@ cDbService::FieldDef cTableRecordings::fields[] =
// name format size index type
// primary key
{ "uuid", ffAscii, 40, fiUuid, ftPrimary },
{ "rec_path", ffAscii, 200, fiRecPath, ftPrimary },
{ "rec_start", ffUInt, 0, fiRecStart, ftPrimary },
//Data
// data
{ "event_id", ffUInt, 0, fiEventId, ftData },
{ "channel_id", ffAscii, 50, fiChannelId, ftData },
{ "scrapinfo_movie_id", ffUInt, 0, fiScrapInfoMovieId, ftData },
@ -848,9 +915,12 @@ cDbService::FieldDef* cTableRecordings::toField(const char* name)
cDbService::IndexDef cTableRecordings::indices[] =
{
// index fields
{ "uuid", { fiUuid, na }, 0 },
{ "rec_path", { fiRecPath, na }, 0 },
{ "uuid", { fiUuid, na }, 0 },
{ "rec_path", { fiRecPath, na }, 0 },
{ "rec_start", { fiRecStart, na }, 0 },
{ "scrap_new", { fiScrapNew, na }, 0 },
{ 0 }
};
};

33
lib/tabledef.h
View File

@ -389,7 +389,7 @@ class cTableVdrs : public cDbTable
public:
cTableVdrs(cDbConnection* aConnection)
: cDbTable(aConnection, fields) { }
: cDbTable(aConnection, fields, indices) { }
virtual const char* TableName() { return "vdrs"; }
@ -413,6 +413,7 @@ class cTableVdrs : public cDbTable
};
static FieldDef fields[];
static IndexDef indices[];
};
//***************************************************************************
@ -526,6 +527,36 @@ class cTableSnapshot : public cDbTable
static IndexDef indices[];
};
//***************************************************************************
// class cTableTimers
//***************************************************************************
class cTableTimers : public cDbTable
{
public:
cTableTimers(cDbConnection* aConnection)
: cDbTable(aConnection, fields) { }
virtual const char* TableName() { return "timers"; }
enum FieldIndex
{
fiEventId,
fiChannelId,
fiVdrUuid,
fiInsSp,
fiUpdSp,
fiState,
fiStartTime,
fiEndTime
};
static FieldDef fields[];
};
//***************************************************************************
// class cTableSeries
//***************************************************************************

96
update.c
View File

@ -188,18 +188,17 @@ int cUpdate::ReadScrapedEvents(void) {
tEvents->getField(cTableEvents::fiScrSp)->name,
lastScrap);
}
select->build(" order by %s", tEvents->getField(cTableEvents::fiInsSp)->name);
status += select->prepare();
if (status != success) {
delete select;
return 0;
}
int eventId = 0;
int seriesId = 0;
int episodeId = 0;
int movieId = 0;
string channelId = "";
int numNew = 0;
for (int res = select->find(); res; res = select->fetch()) {
eventId = tEvents->getIntValue(cTableEvents::fiUseId);
@ -266,23 +265,7 @@ void cUpdate::ReadEpisode(int episodeId, cTVDBSeries *series, string path) {
int status = success;
tEpisodes->clear();
tEpisodes->setValue(cTableSeriesEpisode::fiEpisodeId, episodeId);
cDbStatement *selectEpisode = new cDbStatement(tEpisodes);
selectEpisode->build("select ");
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeName, cDBS::bndOut);
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeNumber, cDBS::bndOut, ", ");
selectEpisode->bind(cTableSeriesEpisode::fiSeasonNumber, cDBS::bndOut, ", ");
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeOverview, cDBS::bndOut, ", ");
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeFirstAired, cDBS::bndOut, ", ");
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeGuestStars, cDBS::bndOut, ", ");
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeRating, cDBS::bndOut, ", ");
selectEpisode->build(" from %s where ", tEpisodes->TableName());
selectEpisode->bind(cTableSeriesEpisode::fiEpisodeId, cDBS::bndIn | cDBS::bndSet);
status += selectEpisode->prepare();
if (status != success) {
delete selectEpisode;
return;
}
int res = selectEpisode->find();
int res = tEpisodes->find();
if (res) {
scrapManager->AddSeriesEpisode(series, tEpisodes);
LoadEpisodeImage(series, episodeId, path);
@ -290,8 +273,6 @@ void cUpdate::ReadEpisode(int episodeId, cTVDBSeries *series, string path) {
if (season > 0)
LoadSeasonPoster(series, season, path);
}
selectEpisode->freeResult();
delete selectEpisode;
return;
}
@ -301,8 +282,9 @@ void cUpdate::LoadEpisodeImage(cTVDBSeries *series, int episodeId, string path)
iPath << path << "/" << "episode_" << episodeId << ".jpg";
string imgPath = iPath.str();
bool imgExists = FileExists(imgPath);
if (!CreateDirectory(path))
return;
if (!imgExists)
if (!CreateDirectory(path))
return;
tSeriesMedia->clear();
cDbValue imageSize;
cDBS::FieldDef imageSizeDef = { "media_content", cDBS::ffUInt, 0, 999, cDBS::ftData };
@ -353,8 +335,9 @@ void cUpdate::LoadSeasonPoster(cTVDBSeries *series, int season, string path) {
string imgPath = iPath.str();
string thumbPath = tPath.str();
bool imgExists = FileExists(imgPath);
if (!CreateDirectory(path))
return;
if (!imgExists)
if (!CreateDirectory(path))
return;
tSeriesMedia->clear();
cDbValue imageSize;
cDBS::FieldDef imageSizeDef = { "media_content", cDBS::ffUInt, 0, 999, cDBS::ftData };
@ -443,8 +426,9 @@ void cUpdate::LoadSeriesActorThumb(cTVDBSeries *series, int actorId, string path
iPath << path << "/" << "actor_" << actorId << ".jpg";
string imgPath = iPath.str();
bool imgExists = FileExists(imgPath);
if (!CreateDirectory(path))
return;
if (!imgExists)
if (!CreateDirectory(path))
return;
cDbValue imageSize;
cDBS::FieldDef imageSizeDef = { "media_content", cDBS::ffUInt, 0, 999, cDBS::ftData };
imageSize.setField(&imageSizeDef);
@ -1193,35 +1177,6 @@ void cUpdate::Action() {
waitCondition.TimedWait(mutex, reconnectTimeout*1000);
continue;
}
//Update Events
if (!config.headless && (forceUpdate || (time(0) - lastScan > scanFreq))) {
if (!init && CheckEpgdBusy())
continue;
int numNewEvents = ReadScrapedEvents();
if (numNewEvents > 0) {
tell(0, "Loaded %d new scraped Events from Database", numNewEvents);
} else {
lastScan = time(0);
forceUpdate = false;
init = false;
continue;
}
tell(0, "Loading new Series and Episodes from Database...");
time_t now = time(0);
int numNewSeries = ReadSeries(false);
int dur = time(0) - now;
tell(0, "Loaded %d new Series and Episodes in %ds from Database", numNewSeries, dur);
//scrapManager->DumpSeries(5);
tell(0, "Loading new Movies from Database...");
now = time(0);
int numNewMovies = ReadMovies(false);
dur = time(0) - now;
tell(0, "Loaded %d new Movies in %ds from Database", numNewMovies, dur);
//scrapManager->DumpMovies(5);
lastScan = time(0);
forceUpdate = false;
}
//Update Recordings from Database
if (forceRecordingUpdate || (time(0) - lastScanNewRecDB > scanNewRecDBFreq)) {
if (!init && CheckEpgdBusy())
@ -1235,6 +1190,35 @@ void cUpdate::Action() {
forceRecordingUpdate = false;
}
//Update Events
if (!config.headless && (forceUpdate || (time(0) - lastScan > scanFreq))) {
if (!init && CheckEpgdBusy())
continue;
int numNewEvents = ReadScrapedEvents();
if (numNewEvents > 0) {
tell(0, "Loaded %d new scraped Events from Database", numNewEvents);
} else {
lastScan = time(0);
forceUpdate = false;
init = false;
continue;
}
tell(0, "Loading new Movies from Database...");
time_t now = time(0);
int numNewMovies = ReadMovies(false);
int dur = time(0) - now;
tell(0, "Loaded %d new Movies in %ds from Database", numNewMovies, dur);
tell(0, "Loading new Series and Episodes from Database...");
now = time(0);
int numNewSeries = ReadSeries(false);
dur = time(0) - now;
tell(0, "Loaded %d new Series and Episodes in %ds from Database", numNewSeries, dur);
lastScan = time(0);
forceUpdate = false;
}
//Scan new recordings
if (init || forceVideoDirUpdate || (time(0) - lastScanNewRec > scanNewRecFreq)) {
if (CheckEpgdBusy()) {